From 9c6fa92b7eed8b29bea96cb73148dba130678e63 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 27 Jan 2026 11:34:17 -0400 Subject: [PATCH 001/188] fix: update RPC publish API fields (#10308) * fix: update purple publish API fields * fix: handle IntegrityError more cleanly * fix: don't import RFC fields from draft * test: update test * chore: remove unused var/import * fix: f-string -> string --- ietf/api/serializers_rpc.py | 29 ++---------------------- ietf/api/tests_views_rpc.py | 44 ++++++++++++++++++------------------- ietf/api/views_rpc.py | 16 +++++++++++++- 3 files changed, 39 insertions(+), 50 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index fe7f6092515..440c2a73d40 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -27,7 +27,7 @@ update_rfcauthors, ) from ietf.group.models import Group -from ietf.name.models import StreamName, StdLevelName, FormalLanguageName +from ietf.name.models import StreamName, StdLevelName from ietf.person.models import Person from ietf.utils import log @@ -137,7 +137,6 @@ class Meta: "pages", "source_format", "authors", - "shepherd", "intended_std_level", "consensus", "shepherd", @@ -263,15 +262,6 @@ class RfcPubSerializer(serializers.ModelSerializer): stream = serializers.PrimaryKeyRelatedField( queryset=StreamName.objects.filter(used=True) ) - formal_languages = serializers.PrimaryKeyRelatedField( - many=True, - required=False, - queryset=FormalLanguageName.objects.filter(used=True), - help_text=( - "formal languages used in RFC (defaults to those from draft, send empty" - "list to override)" - ) - ) std_level = serializers.PrimaryKeyRelatedField( queryset=StdLevelName.objects.filter(used=True), ) @@ -315,11 +305,8 @@ class Meta: "stream", "abstract", "pages", - "words", - "formal_languages", "std_level", "ad", - "note", "obsoletes", "updates", "subseries", @@ -353,9 +340,6 @@ def create(self, validated_data): # If specified, retrieve draft and extract RFC default values from it if draft_name is None: draft = None - defaults_from_draft = { - "group": Group.objects.get(acronym="none", type_id="individ"), - } else: # validation enforces that draft_name and draft_rev are both present draft = Document.objects.filter( @@ -378,17 +362,11 @@ def create(self, validated_data): }, code="already-published-draft", ) - defaults_from_draft = { - "ad": draft.ad, - "formal_languages": draft.formal_languages.all(), - "group": draft.group, - "note": draft.note, - } # Transaction to clean up if something fails with transaction.atomic(): # create rfc, letting validated request data override draft defaults - rfc = self._create_rfc(defaults_from_draft | validated_data) + rfc = self._create_rfc(validated_data) DocEvent.objects.create( doc=rfc, rev=rfc.rev, @@ -523,14 +501,11 @@ def create(self, validated_data): def _create_rfc(self, validated_data): authors_data = validated_data.pop("authors") - formal_languages = validated_data.pop("formal_languages", []) - # todo ad field rfc = Document.objects.create( type_id="rfc", name=f"rfc{validated_data['rfc_number']}", **validated_data, ) - rfc.formal_languages.set(formal_languages) # list of PKs is ok for order, author_data in enumerate(authors_data): rfc.rfcauthor_set.create( order=order, diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index ecb50ee76cd..09fb40bf6e5 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -80,9 +80,15 @@ def test_draftviewset_references(self): def test_notify_rfc_published(self): url = urlreverse("ietf.api.purple_api.notify_rfc_published") area = GroupFactory(type_id="area") + rfc_group = GroupFactory(type_id="wg") draft_ad = RoleFactory(group=area, name_id="ad").person - authors = PersonFactory.create_batch(2) - draft = WgDraftFactory(group__parent=area, authors=authors) + rfc_ad = PersonFactory() + draft_authors = PersonFactory.create_batch(2) + rfc_authors = PersonFactory.create_batch(3) + draft = WgDraftFactory( + group__parent=area, authors=draft_authors, ad=draft_ad, stream_id="ietf" + ) + rfc_stream_id = "ise" assert isinstance(draft, Document), "WgDraftFactory should generate a Document" unused_rfc_number = ( Document.objects.filter(rfc_number__isnull=False).aggregate( @@ -96,7 +102,7 @@ def test_notify_rfc_published(self): "draft_name": draft.name, "draft_rev": draft.rev, "rfc_number": unused_rfc_number, - "title": draft.title, + "title": "RFC " + draft.title, "authors": [ { "titlepage_name": f"titlepage {author.name}", @@ -106,17 +112,14 @@ def test_notify_rfc_published(self): "affiliation": "Some Affiliation", "country": "CA", } - for author in authors + for author in rfc_authors ], - "group": draft.group.acronym, - "stream": draft.stream_id, - "abstract": draft.abstract, - "pages": draft.pages, - "words": draft.pages * 250, - "formal_languages": [], + "group": rfc_group.acronym, + "stream": rfc_stream_id, + "abstract": "RFC version of " + draft.abstract, + "pages": draft.pages + 10, "std_level": "ps", - "ad": draft_ad.pk, - "note": "noted", + "ad": rfc_ad.pk, "obsoletes": [], "updates": [], "subseries": [], @@ -137,7 +140,7 @@ def test_notify_rfc_published(self): ).count(), 1, ) - self.assertEqual(rfc.title, draft.title) + self.assertEqual(rfc.title, "RFC " + draft.title) self.assertEqual(rfc.documentauthor_set.count(), 0) self.assertEqual( list( @@ -159,18 +162,15 @@ def test_notify_rfc_published(self): "affiliation": "Some Affiliation", "country": "CA", } - for author in authors + for author in rfc_authors ], ) - self.assertEqual(rfc.group, draft.group) - self.assertEqual(rfc.stream, draft.stream) - self.assertEqual(rfc.abstract, draft.abstract) - self.assertEqual(rfc.pages, draft.pages) - self.assertEqual(rfc.words, draft.pages * 250) - self.assertEqual(rfc.formal_languages.count(), 0) + self.assertEqual(rfc.group, rfc_group) + self.assertEqual(rfc.stream_id, rfc_stream_id) + self.assertEqual(rfc.abstract, "RFC version of " + draft.abstract) + self.assertEqual(rfc.pages, draft.pages + 10) self.assertEqual(rfc.std_level_id, "ps") - self.assertEqual(rfc.ad, draft_ad) - self.assertEqual(rfc.note, "noted") + self.assertEqual(rfc.ad, rfc_ad) self.assertEqual(rfc.related_that_doc("obs"), []) self.assertEqual(rfc.related_that_doc("updates"), []) self.assertEqual(rfc.part_of(), []) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index ea9c6348cad..6b1799f6544 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -5,6 +5,7 @@ from tempfile import TemporaryDirectory from django.conf import settings +from django.db import IntegrityError from drf_spectacular.utils import OpenApiParameter from rest_framework import mixins, parsers, serializers, viewsets, status from rest_framework.decorators import action @@ -360,7 +361,20 @@ def post(self, request): serializer = RfcPubSerializer(data=request.data) serializer.is_valid(raise_exception=True) # Create RFC - serializer.save() + try: + serializer.save() + except IntegrityError as err: + if Document.objects.filter( + rfc_number=serializer.validated_data["rfc_number"] + ): + raise serializers.ValidationError( + "RFC with that number already exists", + code="rfc-number-in-use", + ) + raise serializers.ValidationError( + f"Unable to publish: {err}", + code="unknown-integrity-error", + ) return Response(NotificationAckSerializer().data) From 33fe0bcb7cafadc2096bfd1386c3d7e8a6a915f5 Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Wed, 28 Jan 2026 13:47:48 -0500 Subject: [PATCH 002/188] feat: add consensus in Draft serializer (#10327) --- ietf/api/serializers_rpc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 440c2a73d40..34e2c791c09 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -175,6 +175,7 @@ class Meta: "pages", "source_format", "authors", + "consensus", ] From a174f43574c5ed4f20ddcc0d600d03e5156fb351 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 30 Jan 2026 14:49:47 -0400 Subject: [PATCH 003/188] fix: use current time for bofreq revisions (#10333) * fix: use current time for bofreq revisions * test: test time handling Adjusts assertion argument order to match our usual style --- ietf/doc/tests_bofreq.py | 19 ++++++++++++------- ietf/doc/views_bofreq.py | 1 - 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ietf/doc/tests_bofreq.py b/ietf/doc/tests_bofreq.py index 6a7c9393ef8..6b142149be0 100644 --- a/ietf/doc/tests_bofreq.py +++ b/ietf/doc/tests_bofreq.py @@ -307,17 +307,20 @@ def test_submit(self): url = urlreverse('ietf.doc.views_bofreq.submit', kwargs=dict(name=doc.name)) rev = doc.rev + doc_time = doc.time r = self.client.post(url,{'bofreq_submission':'enter','bofreq_content':'# oiwefrase'}) self.assertEqual(r.status_code, 302) doc = reload_db_objects(doc) - self.assertEqual(rev, doc.rev) + self.assertEqual(doc.rev, rev) + self.assertEqual(doc.time, doc_time) nobody = PersonFactory() self.client.login(username=nobody.user.username, password=nobody.user.username+'+password') r = self.client.post(url,{'bofreq_submission':'enter','bofreq_content':'# oiwefrase'}) self.assertEqual(r.status_code, 403) doc = reload_db_objects(doc) - self.assertEqual(rev, doc.rev) + self.assertEqual(doc.rev, rev) + self.assertEqual(doc.time, doc_time) self.client.logout() editor = bofreq_editors(doc).first() @@ -339,12 +342,14 @@ def test_submit(self): r = self.client.post(url, postdict) self.assertEqual(r.status_code, 302) doc = reload_db_objects(doc) - self.assertEqual('%02d'%(int(rev)+1) ,doc.rev) - self.assertEqual(f'# {username}', doc.text()) - self.assertEqual(f'# {username}', retrieve_str('bofreq',doc.get_base_name())) - self.assertEqual(docevent_count+1, doc.docevent_set.count()) - self.assertEqual(1, len(outbox)) + self.assertEqual(doc.rev, '%02d'%(int(rev)+1)) + self.assertGreater(doc.time, doc_time) + self.assertEqual(doc.text(), f'# {username}') + self.assertEqual(retrieve_str('bofreq', doc.get_base_name()), f'# {username}') + self.assertEqual(doc.docevent_set.count(), docevent_count+1) + self.assertEqual(len(outbox), 1) rev = doc.rev + doc_time = doc.time finally: os.unlink(file.name) diff --git a/ietf/doc/views_bofreq.py b/ietf/doc/views_bofreq.py index 71cbe304913..94e3960dfa4 100644 --- a/ietf/doc/views_bofreq.py +++ b/ietf/doc/views_bofreq.py @@ -91,7 +91,6 @@ def submit(request, name): by=request.user.person, rev=bofreq.rev, desc='New revision available', - time=bofreq.time, ) bofreq.save_with_history([e]) bofreq_submission = form.cleaned_data['bofreq_submission'] From 2dbe61e891752245447f4c2774353151f0bb34e5 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 9 Feb 2026 14:43:58 -0400 Subject: [PATCH 004/188] feat: speed up agenda.ics; cache more agenda data (#10362) * chore(dev): cprofile middleware settings * feat: precomputed agenda.ics (wip) * feat: precomp filtering support * fix: separately cache versioned hrefs Fixes https://github.com/ietf-tools/datatracker/issues/10355 * fix: versionless agenda href in agenda / ical * fix: preserve RESCHEDULED output * fix: fixup to agree with original output * feat: separate agenda cache, cache old meetings * feat: agenda refresh tasks * chore: undo accidental commit * chore: remove debug parameter * fix: convert session ID to int for comparison * test: update/fix tests, rename new task * refactor: rename task to have _task suffix Also changes a log msg so it won't contain "None" awkwardly * feat: no exceptions from agenda_data_refresh_task * test: explanatory comment * ci: agenda cache for k8s / testcrawl --- ietf/doc/models.py | 16 ++- ietf/meeting/tasks.py | 92 +++++++++++--- ietf/meeting/tests_tasks.py | 64 ++++++++-- ietf/meeting/views.py | 247 +++++++++++++++++++++++++++++++----- ietf/settings.py | 37 +++++- ietf/settings_testcrawl.py | 3 + k8s/settings_local.py | 10 ++ 7 files changed, 396 insertions(+), 73 deletions(-) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index cce9203d09d..463aa6fd977 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -239,14 +239,14 @@ def revisions_by_newrevisionevent(self): return revisions def get_href(self, meeting=None): - return self._get_ref(meeting=meeting,meeting_doc_refs=settings.MEETING_DOC_HREFS) + return self._get_ref(meeting=meeting, versioned=True) def get_versionless_href(self, meeting=None): - return self._get_ref(meeting=meeting,meeting_doc_refs=settings.MEETING_DOC_GREFS) + return self._get_ref(meeting=meeting, versioned=False) - def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): + def _get_ref(self, meeting=None, versioned=True): """ Returns an url to the document text. This differs from .get_absolute_url(), which returns an url to the datatracker page for the document. @@ -255,12 +255,16 @@ def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): # the earlier resolution order, but there's at the moment one single # instance which matches this (with correct results), so we won't # break things all over the place. - if not hasattr(self, '_cached_href'): + cache_attr = "_cached_href" if versioned else "_cached_versionless_href" + if not hasattr(self, cache_attr): validator = URLValidator() if self.external_url and self.external_url.split(':')[0] in validator.schemes: validator(self.external_url) return self.external_url + meeting_doc_refs = ( + settings.MEETING_DOC_HREFS if versioned else settings.MEETING_DOC_GREFS + ) if self.type_id in settings.DOC_HREFS and self.type_id in meeting_doc_refs: if self.meeting_related(): self.is_meeting_related = True @@ -312,8 +316,8 @@ def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): if href.startswith('/'): href = settings.IDTRACKER_BASE_URL + href - self._cached_href = href - return self._cached_href + setattr(self, cache_attr, href) + return getattr(self, cache_attr) def set_state(self, state): """Switch state type implicit in state to state. This just diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py index c361325f9ab..a73763560bf 100644 --- a/ietf/meeting/tasks.py +++ b/ietf/meeting/tasks.py @@ -1,11 +1,14 @@ -# Copyright The IETF Trust 2024-2025, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # import datetime -from celery import shared_task -# from django.db.models import QuerySet +from itertools import batched + +from celery import shared_task, chain +from django.db.models import IntegerField +from django.db.models.functions import Cast from django.utils import timezone from ietf.utils import log @@ -19,9 +22,56 @@ from .utils import fetch_attendance_from_meetings +@shared_task +def agenda_data_refresh_task(num=None): + """Refresh agenda data for one plenary meeting + + If `num` is `None`, refreshes data for the current meeting. + """ + log.log( + f"Refreshing agenda data for {f"IETF-{num}" if num else "current IETF meeting"}" + ) + try: + generate_agenda_data(num, force_refresh=True) + except Exception as err: + # Log and swallow exceptions so failure on one meeting won't break a chain of + # tasks. This is used by agenda_data_refresh_all_task(). + log.log(f"ERROR: Refreshing agenda data failed for num={num}: {err}") + + @shared_task def agenda_data_refresh(): - generate_agenda_data(force_refresh=True) + """Deprecated. Use agenda_data_refresh_task() instead. + + TODO remove this after switching the periodic task to the new name + """ + log.log("Deprecated agenda_data_refresh task called!") + agenda_data_refresh_task() + + +@shared_task +def agenda_data_refresh_all_task(*, batch_size=10): + """Refresh agenda data for all plenary meetings + + Executes as a chain of tasks, each computing up to `batch_size` meetings + in a single task. + """ + meeting_numbers = sorted( + Meeting.objects.annotate( + number_as_int=Cast("number", output_field=IntegerField()) + ) + .filter(type_id="ietf", number_as_int__gt=64) + .values_list("number_as_int", flat=True) + ) + # Batch using chained maps rather than celery.chunk so we only use one worker + # at a time. + batched_task_chain = chain( + *( + agenda_data_refresh_task.map(nums) + for nums in batched(meeting_numbers, batch_size) + ) + ) + batched_task_chain.delay() @shared_task @@ -55,7 +105,9 @@ def proceedings_content_refresh_task(*, all=False): @shared_task def fetch_meeting_attendance_task(): # fetch most recent two meetings - meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by("-date")[:2] + meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by( + "-date" + )[:2] try: stats = fetch_attendance_from_meetings(meetings) except RuntimeError as err: @@ -64,8 +116,11 @@ def fetch_meeting_attendance_task(): for meeting, meeting_stats in zip(meetings, stats): log.log( "Fetched data for meeting {:>3}: {:4d} created, {:4d} updated, {:4d} deleted, {:4d} processed".format( - meeting.number, meeting_stats['created'], meeting_stats['updated'], meeting_stats['deleted'], - meeting_stats['processed'] + meeting.number, + meeting_stats["created"], + meeting_stats["updated"], + meeting_stats["deleted"], + meeting_stats["processed"], ) ) @@ -73,7 +128,7 @@ def fetch_meeting_attendance_task(): def _select_meetings( meetings: list[str] | None = None, meetings_since: str | None = None, - meetings_until: str | None = None + meetings_until: str | None = None, ): # nyah """Select meetings by number or date range""" # IETF-1 = 1986-01-16 @@ -130,15 +185,15 @@ def _select_meetings( @shared_task def resolve_meeting_materials_task( *, # only allow kw arguments - meetings: list[str] | None=None, - meetings_since: str | None=None, - meetings_until: str | None=None + meetings: list[str] | None = None, + meetings_since: str | None = None, + meetings_until: str | None = None, ): """Run materials resolver on meetings - + Can request a set of meetings by number by passing a list in the meetings arg, or by range by passing an iso-format timestamps in meetings_since / meetings_until. - To select all meetings, set meetings_since="zero" and omit other parameters. + To select all meetings, set meetings_since="zero" and omit other parameters. """ meetings_qs = _select_meetings(meetings, meetings_since, meetings_until) for meeting in meetings_qs.order_by("date"): @@ -155,7 +210,9 @@ def resolve_meeting_materials_task( f"meeting {meeting.number}: {err}" ) else: - log.log(f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds.") + log.log( + f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds." + ) @shared_task @@ -163,13 +220,13 @@ def store_meeting_materials_as_blobs_task( *, # only allow kw arguments meetings: list[str] | None = None, meetings_since: str | None = None, - meetings_until: str | None = None + meetings_until: str | None = None, ): """Push meeting materials into the blob store Can request a set of meetings by number by passing a list in the meetings arg, or by range by passing an iso-format timestamps in meetings_since / meetings_until. - To select all meetings, set meetings_since="zero" and omit other parameters. + To select all meetings, set meetings_since="zero" and omit other parameters. """ meetings_qs = _select_meetings(meetings, meetings_since, meetings_until) for meeting in meetings_qs.order_by("date"): @@ -187,4 +244,5 @@ def store_meeting_materials_as_blobs_task( ) else: log.log( - f"Blobs created in {(timezone.now() - mark).total_seconds():0.3f} seconds.") + f"Blobs created in {(timezone.now() - mark).total_seconds():0.3f} seconds." + ) diff --git a/ietf/meeting/tests_tasks.py b/ietf/meeting/tests_tasks.py index a5da00ecbf5..2c5120a39d5 100644 --- a/ietf/meeting/tests_tasks.py +++ b/ietf/meeting/tests_tasks.py @@ -5,23 +5,63 @@ from ietf.utils.test_utils import TestCase from ietf.utils.timezone import date_today from .factories import MeetingFactory -from .tasks import proceedings_content_refresh_task, agenda_data_refresh +from .tasks import ( + proceedings_content_refresh_task, + agenda_data_refresh_task, + agenda_data_refresh_all_task, +) from .tasks import fetch_meeting_attendance_task class TaskTests(TestCase): @patch("ietf.meeting.tasks.generate_agenda_data") - def test_agenda_data_refresh(self, mock_generate): - agenda_data_refresh() + def test_agenda_data_refresh_task(self, mock_generate): + agenda_data_refresh_task() self.assertTrue(mock_generate.called) - self.assertEqual(mock_generate.call_args, call(force_refresh=True)) + self.assertEqual(mock_generate.call_args, call(None, force_refresh=True)) + + mock_generate.reset_mock() + mock_generate.side_effect = RuntimeError + try: + agenda_data_refresh_task() + except Exception as err: + self.fail( + f"agenda_data_refresh_task should not raise exceptions (got {repr(err)})" + ) + + @patch("ietf.meeting.tasks.agenda_data_refresh_task") + @patch("ietf.meeting.tasks.chain") + def test_agenda_data_refresh_all_task(self, mock_chain, mock_agenda_data_refresh): + # Patch the agenda_data_refresh_task task with a mock whose `.map` attribute + # converts its argument, which is expected to be an iterator, to a list + # and returns it. We'll use this to check that the expected task chain + # was set up, but we don't actually run any celery tasks. + mock_agenda_data_refresh.map.side_effect = lambda x: list(x) + + meetings = MeetingFactory.create_batch(5, type_id="ietf") + numbers = sorted(int(m.number) for m in meetings) + agenda_data_refresh_all_task(batch_size=2) + self.assertTrue(mock_chain.called) + # The lists in the call() below are the output of the lambda we patched in + # via mock_agenda_data_refresh.map.side_effect above. I.e., this tests that + # map() was called with the correct batched data. + self.assertEqual( + mock_chain.call_args, + call( + [numbers[0], numbers[1]], + [numbers[2], numbers[3]], + [numbers[4]], + ), + ) + self.assertEqual(mock_agenda_data_refresh.call_count, 0) + self.assertEqual(mock_agenda_data_refresh.map.call_count, 3) @patch("ietf.meeting.tasks.generate_proceedings_content") def test_proceedings_content_refresh_task(self, mock_generate): # Generate a couple of meetings meeting120 = MeetingFactory(type_id="ietf", number="120") # 24 * 5 meeting127 = MeetingFactory(type_id="ietf", number="127") # 24 * 5 + 7 - + # Times to be returned now_utc = datetime.datetime.now(tz=datetime.UTC) hour_00_utc = now_utc.replace(hour=0) @@ -34,19 +74,19 @@ def test_proceedings_content_refresh_task(self, mock_generate): self.assertEqual(mock_generate.call_count, 1) self.assertEqual(mock_generate.call_args, call(meeting120, force_refresh=True)) mock_generate.reset_mock() - + # hour 01 - should call no meetings with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc): proceedings_content_refresh_task() self.assertEqual(mock_generate.call_count, 0) - + # hour 07 - should call meeting with number % 24 == 0 with patch("ietf.meeting.tasks.timezone.now", return_value=hour_07_utc): proceedings_content_refresh_task() self.assertEqual(mock_generate.call_count, 1) self.assertEqual(mock_generate.call_args, call(meeting127, force_refresh=True)) mock_generate.reset_mock() - + # With all=True, all should be called regardless of time. Reuse hour_01_utc which called none before with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc): proceedings_content_refresh_task(all=True) @@ -61,10 +101,10 @@ def test_fetch_meeting_attendance_task(self, mock_fetch_attendance): MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=3)), ] data = { - 'created': 1, - 'updated': 2, - 'deleted': 0, - 'processed': 3, + "created": 1, + "updated": 2, + "deleted": 0, + "processed": 3, } mock_fetch_attendance.return_value = [data, data] diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 903e3c7e79e..8dccda9c877 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -40,7 +40,7 @@ from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile from django.core.validators import URLValidator -from django.urls import reverse,reverse_lazy +from django.urls import reverse, reverse_lazy, NoReverseMatch from django.db.models import F, Max, Q from django.forms.models import modelform_factory, inlineformset_factory from django.template import TemplateDoesNotExist @@ -1859,18 +1859,22 @@ def generate_agenda_data(num=None, force_refresh=False): :num: meeting number :force_refresh: True to force a refresh of the cache """ - cache = caches["default"] - cache_timeout = 6 * 60 - meeting = get_ietf_meeting(num) if meeting is None: raise Http404("No such full IETF meeting") elif int(meeting.number) <= 64: - return Http404("Pre-IETF 64 meetings are not available through this API") - else: - pass + raise Http404("Pre-IETF 64 meetings are not available through this API") + is_current_meeting = meeting.number == get_current_ietf_meeting_num() + + cache = caches["agenda"] + cache_timeout = ( + settings.AGENDA_CACHE_TIMEOUT_CURRENT_MEETING + if is_current_meeting + else settings.AGENDA_CACHE_TIMEOUT_DEFAULT + ) + cache_format = "1" # bump this on backward-incompatible data format changes - cache_key = f"generate_agenda_data_{meeting.number}" + cache_key = f"generate_agenda_data:{meeting.number}:v{cache_format}" if not force_refresh: cached_value = cache.get(cache_key) if cached_value is not None: @@ -1890,8 +1894,6 @@ def generate_agenda_data(num=None, force_refresh=False): filter_organizer = AgendaFilterOrganizer(assignments=filtered_assignments) - is_current_meeting = (num is None) or (num == get_current_ietf_meeting_num()) - # Get Floor Plans floors = FloorPlan.objects.filter(meeting=meeting).order_by('order') @@ -1966,21 +1968,32 @@ def api_get_session_materials(request, session_id=None): ) -def agenda_extract_schedule (item): +def agenda_extract_schedule(item): + if item.session.current_status == "resched": + resched_to = item.session.tombstone_for.official_timeslotassignment() + else: + resched_to = None return { "id": item.id, + "slug": item.slug(), "sessionId": item.session.id, - "room": item.room_name if item.timeslot.show_location else None, + "room": (item.timeslot.get_location() or None) if item.timeslot else None, "location": { "short": item.timeslot.location.floorplan.short, "name": item.timeslot.location.floorplan.name, } if (item.timeslot.show_location and item.timeslot.location and item.timeslot.location.floorplan) else {}, "acronym": item.acronym, - "duration": item.timeslot.duration.seconds, + "duration": item.timeslot.duration.total_seconds(), "name": item.session.name, + "slotId": item.timeslot.id, "slotName": item.timeslot.name, + "slotModified": item.timeslot.modified.isoformat(), "startDateTime": item.timeslot.time.isoformat(), "status": item.session.current_status, + "rescheduledTo": { + "startDateTime": resched_to.timeslot.time.isoformat(), + "duration": resched_to.timeslot.duration.total_seconds(), + } if resched_to is not None else {}, "type": item.session.type.slug, "purpose": item.session.purpose.slug, "isBoF": item.session.group_at_the_time().state_id == "bof", @@ -1998,7 +2011,7 @@ def agenda_extract_schedule (item): "showAgenda": True if (item.session.agenda() is not None or item.session.remote_instructions) else False }, "agenda": { - "url": item.session.agenda().get_href() + "url": item.session.agenda().get_versionless_href() } if item.session.agenda() is not None else { "url": None }, @@ -2290,10 +2303,131 @@ def ical_session_status(assignment): else: return "CONFIRMED" + +def render_icalendar_precomp(agenda_data): + ical_content = generate_agenda_ical_precomp(agenda_data) + return HttpResponse(ical_content, content_type="text/calendar") + + def render_icalendar(schedule, assignments): ical_content = generate_agenda_ical(schedule, assignments) return HttpResponse(ical_content, content_type="text/calendar") + +def generate_agenda_ical_precomp(agenda_data): + """Generate iCalendar from precomputed data using the icalendar library""" + + cal = Calendar() + cal.add("prodid", "-//IETF//datatracker.ietf.org ical agenda//EN") + cal.add("version", "2.0") + cal.add("method", "PUBLISH") + + meeting_data = agenda_data["meeting"] + for item in agenda_data["schedule"]: + event = Event() + + uid = f"ietf-{meeting_data["number"]}-{item["slotId"]}-{item["acronym"]}" + event.add("uid", uid) + + # add custom field with meeting's local TZ + event.add("x-meeting-tz", meeting_data["timezone"]) + + if item["name"]: + summary = item["name"] + else: + summary = f"{item["groupAcronym"]} - {item["groupName"]}" + + if item["note"]: + summary += f" ({item["note"]})" + + event.add("summary", summary) + + if item["room"]: + event.add("location", item["room"]) # room name + + if item["status"] == "canceled": + status = "CANCELLED" + elif item["status"] == "resched": + resched_to = item["rescheduledTo"] + if resched_to is None: + status = "RESCHEDULED" + else: + resched_start = datetime.datetime.fromisoformat( + resched_to["startDateTime"] + ) + dur = datetime.timedelta(seconds=resched_to["duration"]) + resched_end = resched_start + dur + formatted_start = resched_start.strftime("%A %H:%M").upper() + formatted_end = resched_end.strftime("%H:%M") + status = f"RESCHEDULED TO {formatted_start}-{formatted_end}" + else: + status = "CONFIRMED" + event.add("status", status) + + event.add("class", "PUBLIC") + + start_time = datetime.datetime.fromisoformat(item["startDateTime"]) + duration = datetime.timedelta(seconds=item["duration"]) + event.add("dtstart", start_time) + event.add("dtend", start_time + duration) + + # DTSTAMP: when the event was created or last modified (in UTC) + # n.b. timeslot.modified may not be an accurate measure of this + event.add("dtstamp", datetime.datetime.fromisoformat(item["slotModified"])) + + description_parts = [item["slotName"]] + + if item["note"]: + description_parts.append(f"Note: {item["note"]}") + + links = item["links"] + if links["onsiteTool"]: + description_parts.append(f"Onsite tool: {links["onsiteTool"]}") + + if links["videoStream"]: + description_parts.append(f"Meetecho: {links["videoStream"]}") + + if links["webex"]: + description_parts.append(f"Webex: {links["webex"]}") + + if item["remoteInstructions"]: + description_parts.append( + f"Remote instructions: {item["remoteInstructions"]}" + ) + + try: + materials_url = absurl( + "ietf.meeting.views.session_details", + num=meeting_data["number"], + acronym=item["acronym"], + ) + except NoReverseMatch: + pass + else: + description_parts.append(f"Session materials: {materials_url}") + event.add("url", materials_url) + + if meeting_data["number"].isdigit(): + try: + agenda_url = absurl("agenda", num=meeting_data["number"]) + except NoReverseMatch: + pass + else: + description_parts.append(f"See in schedule: {agenda_url}#row-{item["slug"]}") + + if item["agenda"] and item["agenda"]["url"]: + description_parts.append(f"Agenda {item["agenda"]["url"]}") + + # Join all description parts with 2 newlines + description = "\n\n".join(description_parts) + event.add("description", description) + + # Add event to calendar + cal.add_component(event) + + return cal.to_ical().decode("utf-8") + + def generate_agenda_ical(schedule, assignments): """Generate iCalendar using the icalendar library""" @@ -2428,10 +2562,66 @@ def parse_agenda_filter_params(querydict): def should_include_assignment(filter_params, assignment): """Decide whether to include an assignment""" - shown = len(set(filter_params['show']).intersection(assignment.filter_keywords)) > 0 - hidden = len(set(filter_params['hide']).intersection(assignment.filter_keywords)) > 0 + if hasattr(assignment, "filter_keywords"): + kw = assignment.filter_keywords + elif isinstance(assignment, dict): + kw = assignment.get("filterKeywords", []) + else: + raise ValueError("Unsupported assignment instance") + shown = len(set(filter_params['show']).intersection(kw)) > 0 + hidden = len(set(filter_params['hide']).intersection(kw)) > 0 return shown and not hidden + +def agenda_ical_ietf(meeting, filt_params, acronym=None, session_id=None): + agenda_data = generate_agenda_data(meeting.number, force_refresh=False) + if acronym: + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if item["groupAcronym"] == acronym + ] + elif session_id: + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if item["sessionId"] == session_id + ] + if filt_params is not None: + # Apply the filter + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if should_include_assignment(filt_params, item) + ] + return render_icalendar_precomp(agenda_data) + + +def agenda_ical_interim(meeting, filt_params, acronym=None, session_id=None): + schedule = get_schedule(meeting) + + if schedule is None and acronym is None and session_id is None: + raise Http404 + + assignments = SchedTimeSessAssignment.objects.filter( + schedule__in=[schedule, schedule.base], + session__on_agenda=True, + ) + assignments = preprocess_assignments_for_agenda(assignments, meeting) + AgendaKeywordTagger(assignments=assignments).apply() + + if filt_params is not None: + # Apply the filter + assignments = [a for a in assignments if should_include_assignment(filt_params, a)] + + if acronym: + assignments = [ a for a in assignments if a.session.group_at_the_time().acronym == acronym ] + elif session_id: + assignments = [ a for a in assignments if a.session_id == int(session_id) ] + + return render_icalendar(schedule, assignments) + + def agenda_ical(request, num=None, acronym=None, session_id=None): """Agenda ical view @@ -2459,33 +2649,20 @@ def agenda_ical(request, num=None, acronym=None, session_id=None): raise Http404 else: meeting = get_meeting(num, type_in=None) # get requested meeting, whatever its type - schedule = get_schedule(meeting) - if schedule is None and acronym is None and session_id is None: - raise Http404 - - assignments = SchedTimeSessAssignment.objects.filter( - schedule__in=[schedule, schedule.base], - session__on_agenda=True, - ) - assignments = preprocess_assignments_for_agenda(assignments, meeting) - AgendaKeywordTagger(assignments=assignments).apply() + if isinstance(session_id, str) and session_id.isdigit(): + session_id = int(session_id) try: filt_params = parse_agenda_filter_params(request.GET) except ValueError as e: return HttpResponseBadRequest(str(e)) - if filt_params is not None: - # Apply the filter - assignments = [a for a in assignments if should_include_assignment(filt_params, a)] - - if acronym: - assignments = [ a for a in assignments if a.session.group_at_the_time().acronym == acronym ] - elif session_id: - assignments = [ a for a in assignments if a.session_id == int(session_id) ] + if meeting.type_id == "ietf": + return agenda_ical_ietf(meeting, filt_params, acronym, session_id) + else: + return agenda_ical_interim(meeting, filt_params, acronym, session_id) - return render_icalendar(schedule, assignments) @cache_page(15 * 60) def agenda_json(request, num=None): diff --git a/ietf/settings.py b/ietf/settings.py index 1cda79e21b0..899a377ad74 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -228,6 +228,10 @@ BLOBSTORAGE_CONNECT_TIMEOUT = 10 # seconds; boto3 default is 60 BLOBSTORAGE_READ_TIMEOUT = 10 # seconds; boto3 default is 60 +# Caching for agenda data in seconds +AGENDA_CACHE_TIMEOUT_DEFAULT = 8 * 24 * 60 * 60 # 8 days +AGENDA_CACHE_TIMEOUT_CURRENT_MEETING = 6 * 60 # 6 minutes + WSGI_APPLICATION = "ietf.wsgi.application" AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', ) @@ -1400,6 +1404,16 @@ def skip_unreadable_post(record): f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), }, + "agenda": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, "proceedings": { "BACKEND": "ietf.utils.cache.LenientMemcacheCache", "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", @@ -1453,6 +1467,17 @@ def skip_unreadable_post(record): "VERSION": __version__, "KEY_PREFIX": "ietf:dt", }, + "agenda": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + # "LOCATION": "127.0.0.1:11211", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, "proceedings": { "BACKEND": "django.core.cache.backends.dummy.DummyCache", # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", @@ -1519,11 +1544,17 @@ def skip_unreadable_post(record): NOMCOM_APP_SECRET = b'\x9b\xdas1\xec\xd5\xa0SI~\xcb\xd4\xf5t\x99\xc4i\xd7\x9f\x0b\xa9\xe8\xfeY\x80$\x1e\x12tN:\x84' ALLOWED_HOSTS = ['*',] - + try: # see https://github.com/omarish/django-cprofile-middleware - import django_cprofile_middleware # pyflakes:ignore - MIDDLEWARE = MIDDLEWARE + ['django_cprofile_middleware.middleware.ProfilerMiddleware', ] + import django_cprofile_middleware # pyflakes:ignore + + MIDDLEWARE = MIDDLEWARE + [ + "django_cprofile_middleware.middleware.ProfilerMiddleware", + ] + DJANGO_CPROFILE_MIDDLEWARE_REQUIRE_STAFF = ( + False # Do not use this setting for a public site! + ) except ImportError: pass diff --git a/ietf/settings_testcrawl.py b/ietf/settings_testcrawl.py index 40744a228df..edb978757a9 100644 --- a/ietf/settings_testcrawl.py +++ b/ietf/settings_testcrawl.py @@ -27,6 +27,9 @@ 'MAX_ENTRIES': 10000, }, }, + 'agenda': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, 'proceedings': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', }, diff --git a/k8s/settings_local.py b/k8s/settings_local.py index f8ffacc83fc..0386dbbdf9c 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -306,6 +306,16 @@ def _multiline_to_list(s): f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), }, + "agenda": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, "proceedings": { "BACKEND": "ietf.utils.cache.LenientMemcacheCache", "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", From f8be1436fb9570eb623c6408cbb755e39bb18b22 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 10 Feb 2026 13:33:39 -0400 Subject: [PATCH 005/188] fix: add id attr to liaisons ButtonWidget (#10389) --- ietf/liaisons/widgets.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ietf/liaisons/widgets.py b/ietf/liaisons/widgets.py index 74368e83f2b..48db8af0a31 100644 --- a/ietf/liaisons/widgets.py +++ b/ietf/liaisons/widgets.py @@ -26,7 +26,9 @@ def render(self, name, value, **kwargs): html += '%s' % conditional_escape(i) required_str = 'Please fill in %s to attach a new file' % conditional_escape(self.required_label) html += '%s' % conditional_escape(required_str) - html += '' % conditional_escape(self.label) + html += ''.format( + f"id_{name}", conditional_escape(self.label) + ) return mark_safe(html) From 4a024d9d64e36714523b2f9e04bdbc3005aefc03 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 10 Feb 2026 12:25:43 -0600 Subject: [PATCH 006/188] fix: identify editorial drafts that should not expire (#10388) --- ietf/doc/expire.py | 50 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py index bf8523aa98f..d42af628f80 100644 --- a/ietf/doc/expire.py +++ b/ietf/doc/expire.py @@ -38,22 +38,46 @@ def expirable_drafts(queryset=None): # Populate this first time through (but after django has been set up) if nonexpirable_states is None: # all IESG states except I-D Exists and Dead block expiry - nonexpirable_states = list(State.objects.filter(used=True, type="draft-iesg").exclude(slug__in=("idexists", "dead"))) + nonexpirable_states = list( + State.objects.filter(used=True, type="draft-iesg").exclude( + slug__in=("idexists", "dead") + ) + ) # sent to RFC Editor and RFC Published block expiry (the latter # shouldn't be possible for an active draft, though) - nonexpirable_states += list(State.objects.filter(used=True, type__in=("draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"), slug__in=("rfc-edit", "pub"))) + nonexpirable_states += list( + State.objects.filter( + used=True, + type__in=( + "draft-stream-iab", + "draft-stream-irtf", + "draft-stream-ise", + "draft-stream-editorial", + ), + slug__in=("rfc-edit", "pub"), + ) + ) # other IRTF states that block expiration - nonexpirable_states += list(State.objects.filter(used=True, type_id="draft-stream-irtf", slug__in=("irsgpoll", "iesg-rev",))) - - return queryset.filter( - states__type="draft", states__slug="active" - ).exclude( - expires=None - ).exclude( - states__in=nonexpirable_states - ).exclude( - tags="rfc-rev" # under review by the RFC Editor blocks expiry - ).distinct() + nonexpirable_states += list( + State.objects.filter( + used=True, + type_id="draft-stream-irtf", + slug__in=( + "irsgpoll", + "iesg-rev", + ), + ) + ) + + return ( + queryset.filter(states__type="draft", states__slug="active") + .exclude(expires=None) + .exclude(states__in=nonexpirable_states) + .exclude( + tags="rfc-rev" # under review by the RFC Editor blocks expiry + ) + .distinct() + ) def get_soon_to_expire_drafts(days_of_warning): From 832c62e5c1f0cba736842cbe241236bfc8cf386f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 11 Feb 2026 11:41:58 -0400 Subject: [PATCH 007/188] feat: format+has_errata, drop see_also for red API (#10392) * fix: has_errata field for red API * chore: explanatory comment * feat: accurate format list for red API * refactor: specify blob names in API * chore: remove see_also field * fix: finish removing see_also field --- ietf/doc/api.py | 27 ++++++++++++++++++++++----- ietf/doc/models.py | 18 ++++++++++++++++++ ietf/doc/serializers.py | 33 +++++++++++++++++++++++---------- 3 files changed, 63 insertions(+), 15 deletions(-) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 47e7e6fffdb..6a4c0c9fd5b 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -1,7 +1,17 @@ # Copyright The IETF Trust 2024-2026, All Rights Reserved """Doc API implementations""" -from django.db.models import OuterRef, Subquery, Prefetch, Value, JSONField, QuerySet +from django.db.models import ( + BooleanField, + Count, + JSONField, + OuterRef, + Prefetch, + Q, + QuerySet, + Subquery, + Value, +) from django.db.models.functions import TruncDate from django_filters import rest_framework as filters from rest_framework import filters as drf_filters @@ -133,11 +143,18 @@ def augment_rfc_queryset(queryset: QuerySet[Document]): ) .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) .annotate( - # TODO implement these fake fields for real - see_also=Value([], output_field=JSONField()), - formats=Value(["txt", "xml"], output_field=JSONField()), + # Count of "verified-errata" tags will be 1 or 0, convert to Boolean + has_errata=Count( + "tags", + filter=Q( + tags__slug="verified-errata", + ), + output_field=BooleanField(), + ) + ) + .annotate( + # TODO implement this fake field for real keywords=Value(["keyword"], output_field=JSONField()), - errata=Value([], output_field=JSONField()), ) ) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 463aa6fd977..ec9a25add82 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1284,6 +1284,24 @@ def action_holders_enabled(self): iesg_state = self.get_state('draft-iesg') return iesg_state and iesg_state.slug != 'idexists' + def formats(self): + """List of file formats available + + Only implemented for RFCs. Relies on StoredObject. + """ + if self.type_id != "rfc": + raise RuntimeError("Only allowed for type=rfc") + return [ + { + "fmt": Path(object_name).parts[0], + "name": object_name, + } + for object_name in StoredObject.objects.filter( + store="rfc", doc_name=self.name, doc_rev=self.rev + ).values_list("name", flat=True) + ] + + class DocumentURL(models.Model): doc = ForeignKey(Document) tag = ForeignKey(DocUrlTagName) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index 05647d9ce12..e8d373164ba 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -16,6 +16,7 @@ class RfcAuthorSerializer(serializers.ModelSerializer): """Serializer for an RfcAuthor / DocumentAuthor in a response""" + datatracker_person_path = serializers.URLField( source="person.get_absolute_url", required=False, @@ -36,7 +37,7 @@ class Meta: def to_representation(self, instance): """instance -> primitive data types - + Translates a DocumentAuthor into an equivalent RfcAuthor we can use the same serializer for either type. """ @@ -87,7 +88,15 @@ class DocIdentifierSerializer(serializers.Serializer): type RfcStatusSlugT = Literal[ - "std", "ps", "ds", "bcp", "inf", "exp", "hist", "unkn", "not-issued", + "std", + "ps", + "ds", + "bcp", + "inf", + "exp", + "hist", + "unkn", + "not-issued", ] @@ -188,11 +197,16 @@ class ContainingSubseriesSerializer(serializers.Serializer): type = serializers.CharField(source="source.type_id") +class RfcFormatSerializer(serializers.Serializer): + RFC_FORMATS = ("xml", "txt", "html", "pdf", "ps", "json", "notprepped") + + fmt = serializers.ChoiceField(choices=RFC_FORMATS) + name = serializers.CharField(help_text="Name of blob in the blob store") + + class RfcMetadataSerializer(serializers.ModelSerializer): """Serialize metadata of an RFC""" - RFC_FORMATS = ("xml", "txt", "html", "htmlized", "pdf", "ps") - number = serializers.IntegerField(source="rfc_number") published = serializers.DateField() status = RfcStatusSerializer(source="*") @@ -207,10 +221,11 @@ class RfcMetadataSerializer(serializers.ModelSerializer): updates = RelatedRfcSerializer(many=True, read_only=True) updated_by = ReverseRelatedRfcSerializer(many=True, read_only=True) subseries = ContainingSubseriesSerializer(many=True, read_only=True) - see_also = serializers.ListField(child=serializers.CharField(), read_only=True) - formats = serializers.MultipleChoiceField(choices=RFC_FORMATS) + formats = RfcFormatSerializer( + many=True, read_only=True, help_text="Available formats" + ) keywords = serializers.ListField(child=serializers.CharField(), read_only=True) - errata = serializers.ListField(child=serializers.CharField(), read_only=True) + has_errata = serializers.BooleanField(read_only=True) class Meta: model = Document @@ -230,15 +245,13 @@ class Meta: "updates", "updated_by", "subseries", - "see_also", "draft", "abstract", "formats", "keywords", - "errata", + "has_errata", ] - @extend_schema_field(RfcAuthorSerializer(many=True)) def get_authors(self, doc: Document): # If doc has any RfcAuthors, use those, otherwise fall back to DocumentAuthors From 66a1bf0a9ecae30036cf23fc07f6f26b27b1e94d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 11 Feb 2026 15:00:57 -0400 Subject: [PATCH 008/188] chore: un-squelch bibtexparser DeprecationWarnings (#10395) --- ietf/settings.py | 2 -- requirements.txt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index 899a377ad74..565e8825a9f 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -36,8 +36,6 @@ warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="oic.utils.time_util") warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="pytz.tzinfo") warnings.filterwarnings("ignore", message="'instantiateVariableFont' is deprecated", module="weasyprint") -warnings.filterwarnings("ignore", category=DeprecationWarning, module="bibtexparser") # https://github.com/sciunto-org/python-bibtexparser/issues/502 -warnings.filterwarnings("ignore", category=DeprecationWarning, module="pyparsing") # https://github.com/sciunto-org/python-bibtexparser/issues/502 base_path = pathlib.Path(__file__).resolve().parent diff --git a/requirements.txt b/requirements.txt index 3f89f6f16c1..cb583d5dc97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ setuptools>=80.9.0 # Require this first, to prevent later errors aiosmtpd>=1.4.6 argon2-cffi>=25.1.0 # For the Argon2 password hasher option beautifulsoup4>=4.13.4 # Only used in tests -bibtexparser>=1.4.3 # Only used in tests +bibtexparser>=1.4.4 # Only used in tests bleach>=6.2.0 # project is deprecated but supported types-bleach>=6.2.0 boto3>=1.39.15 From 0b637ef4ace72f31a963b7603a8f69a253420810 Mon Sep 17 00:00:00 2001 From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com> Date: Wed, 11 Feb 2026 19:13:44 +0000 Subject: [PATCH 009/188] ci: update base image target version to 20260211T1901 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 41ff295eecf..71370fabee5 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260114T1756 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260211T1901 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 3ad31c7e25d..947f3790e4d 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260114T1756 +20260211T1901 From 492888b8a22113becc2cbe3900ba3294cfc6d7f6 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 12 Feb 2026 11:34:11 -0400 Subject: [PATCH 010/188] fix: handle doc_rev is None in Document.formats (#10401) Likely a temporary fix, but safer for quick deployment while we work on the bigger project. --- ietf/api/views_rpc.py | 2 +- ietf/doc/models.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 6b1799f6544..2bf16480f2b 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -487,7 +487,7 @@ def post(self, request): name=self._blob_destination(ftm), file=f, doc_name=rfc.name, - doc_rev=rfc.rev, # expect None, but match whatever it is + doc_rev=rfc.rev, # expect blank, but match whatever it is mtime=mtime, ) destination = self._fs_destination(ftm) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index ec9a25add82..8f700bf496b 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -12,6 +12,8 @@ from io import BufferedReader from pathlib import Path + +from django.db.models import Q from lxml import etree from typing import Optional, Protocol, TYPE_CHECKING, Union from weasyprint import HTML as wpHTML @@ -1291,13 +1293,21 @@ def formats(self): """ if self.type_id != "rfc": raise RuntimeError("Only allowed for type=rfc") + + # StoredObject.doc_rev can be null or "" to represent no rev. Match either + # of these when self.rev is "" (always expected to be the case for RFCs) + rev_q = Q(doc_rev=self.rev) + if self.rev == "": + rev_q |= Q(doc_rev__isnull=True) return [ { "fmt": Path(object_name).parts[0], "name": object_name, } for object_name in StoredObject.objects.filter( - store="rfc", doc_name=self.name, doc_rev=self.rev + rev_q, + store="rfc", + doc_name=self.name, ).values_list("name", flat=True) ] From 1b306eb7f3b9b45b3162101c7fc216ac7da3eab1 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 12 Feb 2026 13:49:22 -0400 Subject: [PATCH 011/188] feat: bofreq timestamp fixup task (#10402) * feat: utility to fix up bofreq timestamps * fix: don't fix -00 (+ logging) * feat: task * chore: disable test coverage for one-off task --- ietf/doc/tasks.py | 8 ++- ietf/doc/utils_bofreq.py | 143 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 147 insertions(+), 4 deletions(-) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index 02b7c2a07d9..b463b9cecf2 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2024-2025, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # @@ -34,6 +34,7 @@ ensure_draft_bibxml_path_exists, investigate_fragment, ) +from .utils_bofreq import fixup_bofreq_timestamps @shared_task @@ -149,3 +150,8 @@ def rebuild_reference_relations_task(doc_names: list[str]): rebuild_reference_relations(doc, filenames) else: log.log(f"Found no content for {stem}") + + +@shared_task +def fixup_bofreq_timestamps_task(): # pragma: nocover + fixup_bofreq_timestamps() diff --git a/ietf/doc/utils_bofreq.py b/ietf/doc/utils_bofreq.py index aec8f60ad67..d01b039b8e8 100644 --- a/ietf/doc/utils_bofreq.py +++ b/ietf/doc/utils_bofreq.py @@ -1,12 +1,149 @@ -# Copyright The IETF Trust 2021 All Rights Reserved +# Copyright The IETF Trust 2021-2026 All Rights Reserved +import datetime +from pathlib import Path -from ietf.doc.models import BofreqEditorDocEvent, BofreqResponsibleDocEvent +from django.conf import settings + +from ietf.doc.models import ( + BofreqEditorDocEvent, + BofreqResponsibleDocEvent, + DocEvent, + DocHistory, + Document, +) from ietf.person.models import Person +from ietf.utils import log + def bofreq_editors(bofreq): e = bofreq.latest_event(BofreqEditorDocEvent) return e.editors.all() if e else Person.objects.none() + def bofreq_responsible(bofreq): e = bofreq.latest_event(BofreqResponsibleDocEvent) - return e.responsible.all() if e else Person.objects.none() \ No newline at end of file + return e.responsible.all() if e else Person.objects.none() + + +def fixup_bofreq_timestamps(): # pragma: nocover + """Fixes bofreq event / document timestamps + + Timestamp errors resulted from the bug fixed by + https://github.com/ietf-tools/datatracker/pull/10333 + + Does not fix up -00 revs because the timestamps on these were not affected by + the bug. Replacing their timestamps creates a confusing event history because the + filesystem timestamp is usually a fraction of a second later than other events + created upon the initial rev creation. This causes the "New revision available" + event to appear _after_ these events in the history. Better to leave them as is. + """ + FIX_DEPLOYMENT_TIME = "2026-02-03T01:16:00+00:00" # 12.58.0 -> production + + def _get_doc_time(doc_name: str, rev: str): + path = Path(settings.BOFREQ_PATH) / f"{doc_name}-{rev}.md" + return datetime.datetime.fromtimestamp(path.stat().st_mtime, datetime.UTC) + + # Find affected DocEvents and DocHistories + new_bofreq_events = ( + DocEvent.objects.filter( + doc__type="bofreq", type="new_revision", time__lt=FIX_DEPLOYMENT_TIME + ) + .exclude(rev="00") # bug did not affect rev 00 events + .order_by("doc__name", "rev") + ) + log.log( + f"fixup_bofreq_timestamps: found {new_bofreq_events.count()} " + f"new_revision events before {FIX_DEPLOYMENT_TIME}" + ) + document_fixups = {} + for e in new_bofreq_events: + name = e.doc.name + rev = e.rev + filesystem_time = _get_doc_time(name, rev) + assert e.time < filesystem_time, ( + f"Rev {rev} event timestamp for {name} unexpectedly later than the " + "filesystem timestamp!" + ) + try: + dochistory = DocHistory.objects.filter( + name=name, time__lt=filesystem_time + ).get(rev=rev) + except DocHistory.MultipleObjectsReturned as err: + raise RuntimeError( + f"Multiple DocHistories for {name} rev {rev} exist earlier than the " + "filesystem timestamp!" + ) from err + except DocHistory.DoesNotExist as err: + if rev == "00": + # Unreachable because we don't adjust -00 revs, but could be needed + # if we did, in theory. In practice it's still not reached, but + # keeping the case for completeness. + dochistory = None + else: + raise RuntimeError( + f"No DocHistory for {name} rev {rev} exists earlier than the " + f"filesystem timestamp!" + ) from err + + if name not in document_fixups: + document_fixups[name] = [] + document_fixups[name].append( + { + "event": e, + "dochistory": dochistory, + "filesystem_time": filesystem_time, + } + ) + + # Now do the actual fixup + system_person = Person.objects.get(name="(System)") + for doc_name, fixups in document_fixups.items(): + bofreq = Document.objects.get(type="bofreq", name=doc_name) + log_msg_parts = [] + adjusted_revs = [] + for fixup in fixups: + event_to_fix = fixup["event"] + dh_to_fix = fixup["dochistory"] + new_time = fixup["filesystem_time"] + adjusted_revs.append(event_to_fix.rev) + + # Fix up the event + event_to_fix.time = new_time + event_to_fix.save() + log_msg_parts.append(f"rev {event_to_fix.rev} DocEvent") + + # Fix up the DocHistory + if dh_to_fix is not None: + dh_to_fix.time = new_time + dh_to_fix.save() + log_msg_parts.append(f"rev {dh_to_fix.rev} DocHistory") + + if event_to_fix.rev == bofreq.rev and bofreq.time < new_time: + # Update the Document without calling save(). Only update if + # the time has not changed so we don't inadvertently overwrite + # a concurrent update. + Document.objects.filter(pk=bofreq.pk, time=bofreq.time).update( + time=new_time + ) + bofreq.refresh_from_db() + if bofreq.rev == event_to_fix.rev: + log_msg_parts.append(f"rev {bofreq.rev} Document") + else: + log.log( + "fixup_bofreq_timestamps: WARNING: bofreq Document rev " + f"changed for {bofreq.name}" + ) + log.log(f"fixup_bofreq_timestamps: {bofreq.name}: " + ", ".join(log_msg_parts)) + + # Fix up the Document, if necessary, and add a record of the adjustment + DocEvent.objects.create( + type="added_comment", + by=system_person, + doc=bofreq, + rev=bofreq.rev, + desc=( + "Corrected inaccurate document and new revision event timestamps for " + + ("version " if len(adjusted_revs) == 1 else "versions ") + + ", ".join(adjusted_revs) + ), + ) From 4945809b7804c1f15de1b0340be269dd7e200f95 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 12 Feb 2026 13:56:46 -0400 Subject: [PATCH 012/188] chore(dev): update beat in docker-compose.yml (#10330) Fixes commented-out config --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2440faf1214..ebe53cf95a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -129,7 +129,7 @@ services: # but can be enabled by uncommenting the following. # # beat: -# image: ghcr.io/ietf-tools/datatracker-celery:latest +# image: "${COMPOSE_PROJECT_NAME}-celery" # init: true # environment: # CELERY_APP: ietf From 8005a8baa6ffb72c47d6e35f44c0e5d78b456a2d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 13 Feb 2026 11:36:23 -0400 Subject: [PATCH 013/188] chore(dev): update docker-compose depends_on (#10410) * chore(dev): update docker-compose depends_on * chore(dev): another depends_on tweak app/celery don't actually use the blobstore container, but the Django config refers to it so we should probably depend on it anyway --- docker-compose.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ebe53cf95a8..4c3f2f6b8ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,9 +13,10 @@ services: # network_mode: service:db depends_on: + - blobdb + - blobstore - db - mq - - blobstore ipc: host @@ -79,7 +80,10 @@ services: command: - '--loglevel=INFO' depends_on: + - blobdb + - blobstore - db + - mq restart: unless-stopped stop_grace_period: 1m volumes: @@ -102,7 +106,10 @@ services: - '--concurrency=1' depends_on: + - blobdb + - blobstore - db + - mq restart: unless-stopped stop_grace_period: 1m volumes: From 8d804f3427b4d4c40aa6bfadba92a433bd468b26 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 20 Feb 2026 15:22:24 -0400 Subject: [PATCH 014/188] feat: button to push slide decks to Meetecho (#10431) * refactor: eliminate inline script Partially removes jQuery from this corner * chore: indicate whether slide updates were sent * feat: admin button to push slide decks to Meetecho * test: new test * test: cover interim case --- ietf/meeting/tests_views.py | 63 +++++++++++++++- ietf/meeting/urls.py | 3 +- ietf/meeting/views.py | 46 ++++++++++++ ietf/static/js/session_details.js | 53 ++++++++++++++ ietf/templates/meeting/session_details.html | 81 ++++++--------------- ietf/utils/meetecho.py | 48 +++++++++--- ietf/utils/tests_meetecho.py | 47 ++++++++++-- package.json | 1 + 8 files changed, 268 insertions(+), 74 deletions(-) create mode 100644 ietf/static/js/session_details.js diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index b94229d9693..168999d0aa7 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -4754,7 +4754,7 @@ def _approval_url(slidesub): 0, "second session proposed slides should be linked for approval", ) - + class EditScheduleListTests(TestCase): def setUp(self): @@ -7345,6 +7345,67 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls): fd.close() self.assertIn('third version', contents) + @override_settings( + MEETECHO_API_CONFIG="fake settings" + ) # enough to trigger API calls + @patch("ietf.meeting.views.SlidesManager") + def test_notify_meetecho_of_all_slides(self, mock_slides_manager_cls): + for meeting_type in ["ietf", "interim"]: + # Reset for the sake of the second iteration + self.client.logout() + mock_slides_manager_cls.reset_mock() + + session = SessionFactory(meeting__type_id=meeting_type) + meeting = session.meeting + + # bad meeting + url = urlreverse( + "ietf.meeting.views.notify_meetecho_of_all_slides", + kwargs={"num": 9999, "acronym": session.group.acronym}, + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + r = self.client.post(url) + self.assertEqual(r.status_code, 404) + self.assertFalse(mock_slides_manager_cls.called) + self.client.logout() + + # good meeting + url = urlreverse( + "ietf.meeting.views.notify_meetecho_of_all_slides", + kwargs={"num": meeting.number, "acronym": session.group.acronym}, + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 405) + self.assertFalse(mock_slides_manager_cls.called) + mock_slides_manager = mock_slides_manager_cls.return_value + mock_slides_manager.send_update.return_value = True + r = self.client.post(url) + self.assertEqual(r.status_code, 302) + self.assertEqual(mock_slides_manager.send_update.call_count, 1) + self.assertEqual(mock_slides_manager.send_update.call_args, call(session)) + r = self.client.get(r["Location"]) + messages = list(r.context["messages"]) + self.assertEqual(len(messages), 1) + self.assertEqual( + str(messages[0]), f"Notified Meetecho about slides for {session}" + ) + + mock_slides_manager.send_update.reset_mock() + mock_slides_manager.send_update.return_value = False + r = self.client.post(url) + self.assertEqual(r.status_code, 302) + self.assertEqual(mock_slides_manager.send_update.call_count, 1) + self.assertEqual(mock_slides_manager.send_update.call_args, call(session)) + r = self.client.get(r["Location"]) + messages = list(r.context["messages"]) + self.assertEqual(len(messages), 1) + self.assertIn( + "No sessions were eligible for Meetecho slides update.", str(messages[0]) + ) + @override_settings(IETF_NOTES_URL='https://notes.ietf.org/') class ImportNotesTests(TestCase): diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py index af36a6656c3..a038e1cfe6e 100644 --- a/ietf/meeting/urls.py +++ b/ietf/meeting/urls.py @@ -15,6 +15,7 @@ def get_redirect_url(self, *args, **kwargs): safe_for_all_meeting_types = [ url(r'^session/(?P[-a-z0-9]+)/?$', views.session_details), + url(r'^session/(?P[-a-z0-9]+)/send_slide_notifications$', views.notify_meetecho_of_all_slides), url(r'^session/(?P\d+)/drafts$', views.add_session_drafts), url(r'^session/(?P\d+)/recordings$', views.add_session_recordings), url(r'^session/(?P\d+)/attendance$', views.session_attendance), @@ -30,7 +31,7 @@ def get_redirect_url(self, *args, **kwargs): url(r'^session/(?P\d+)/doc/%(name)s/remove$' % settings.URL_REGEXPS, views.remove_sessionpresentation), url(r'^session/(?P\d+)\.ics$', views.agenda_ical), url(r'^sessions/(?P[-a-z0-9]+)\.ics$', views.agenda_ical), - url(r'^slidesubmission/(?P\d+)$', views.approve_proposed_slides) + url(r'^slidesubmission/(?P\d+)$', views.approve_proposed_slides), ] diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 8dccda9c877..731dfad88fb 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -5710,6 +5710,52 @@ def approve_proposed_slides(request, slidesubmission_id, num): }) +@role_required("Secretariat") +def notify_meetecho_of_all_slides(request, num, acronym): + """Notify meetecho of state of all slides for the group + + Respects the usual notification window around each session. Meetecho will ignore + notices outside that window anyway, so no sense sending them. + """ + meeting = get_meeting(num=num, type_in=None) # raises 404 + if request.method != "POST": + return HttpResponseNotAllowed( + content="Method not allowed", + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + permitted_methods=("POST",), + ) + scheduled_sessions = [ + session + for session in get_sessions(meeting.number, acronym) + if session.current_status == "sched" + ] + sm = SlidesManager(api_config=settings.MEETECHO_API_CONFIG) + updated = [] + for session in scheduled_sessions: + if sm.send_update(session): + updated.append(session) + if len(updated) > 0: + messages.success( + request, + f"Notified Meetecho about slides for {','.join(str(s) for s in updated)}", + ) + elif sm.slides_notify_time is not None: + messages.warning( + request, + "No sessions were eligible for Meetecho slides update. Updates are " + f"only sent within {sm.slides_notify_time} before or after the session.", + ) + else: + messages.warning( + request, + "No sessions were eligible for Meetecho slides update. Updates are " + "currently disabled.", + ) + return redirect( + "ietf.meeting.views.session_details", num=meeting.number, acronym=acronym + ) + + def import_session_minutes(request, session_id, num): """Import session minutes from the ietf.notes.org site diff --git a/ietf/static/js/session_details.js b/ietf/static/js/session_details.js new file mode 100644 index 00000000000..03d1b2d3d91 --- /dev/null +++ b/ietf/static/js/session_details.js @@ -0,0 +1,53 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +// Relies on other scripts being loaded, see usage in session_details.html +document.addEventListener('DOMContentLoaded', () => { + // Init with best guess at local timezone. + ietf_timezone.set_tz_change_callback(timezone_changed) // cb is in upcoming.js + ietf_timezone.initialize('local') + + // Set up sortable elements if the user can manage materials + if (document.getElementById('can-manage-materials-flag')) { + const sortables = [] + const options = { + group: 'slides', + animation: 150, + handle: '.drag-handle', + onAdd: function (event) {onAdd(event)}, + onRemove: function (event) {onRemove(event)}, + onEnd: function (event) {onEnd(event)} + } + + function onAdd (event) { + const old_session = event.from.getAttribute('data-session') + const new_session = event.to.getAttribute('data-session') + $.post(event.to.getAttribute('data-add-to-session'), { + 'order': event.newIndex + 1, + 'name': event.item.getAttribute('name') + }) + $(event.item).find('td:eq(1)').find('a').each(function () { + $(this).attr('href', $(this).attr('href').replace(old_session, new_session)) + }) + } + + function onRemove (event) { + const old_session = event.from.getAttribute('data-session') + $.post(event.from.getAttribute('data-remove-from-session'), { + 'oldIndex': event.oldIndex + 1, + 'name': event.item.getAttribute('name') + }) + } + + function onEnd (event) { + if (event.to == event.from) { + $.post(event.from.getAttribute('data-reorder-in-session'), { + 'oldIndex': event.oldIndex + 1, + 'newIndex': event.newIndex + 1 + }) + } + } + + for (const elt of document.querySelectorAll('.slides tbody')) { + sortables.push(Sortable.create(elt, options)) + } + } +}) diff --git a/ietf/templates/meeting/session_details.html b/ietf/templates/meeting/session_details.html index 55fa3d38571..a4d9ba10904 100644 --- a/ietf/templates/meeting/session_details.html +++ b/ietf/templates/meeting/session_details.html @@ -1,5 +1,5 @@ {% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin ietf_filters static %} {% block title %}{{ meeting }} : {{ group.acronym }}{% endblock %} {% block morecss %} @@ -53,69 +53,36 @@

Unscheduled Sessions

{% endif %} {% if forloop.last %}{% endif %} {% endfor %} + {% if user|has_role:"Secretariat" %} +
+
+ Secretariat Only +
+
+
+ {% csrf_token %} + +
+
+
+ {% endif %} + {% comment %} + The existence of an element with id canManageMaterialsFlag is checked in + session_details.js to determine whether it should init the sortable tables. + Not the most elegant approach, but it works. + {% endcomment %} + {% if can_manage_materials %}
{% endif %} {% endblock %} {% block js %} - {% if can_manage_materials %} - {% endif %} + {% endblock %} \ No newline at end of file diff --git a/ietf/utils/meetecho.py b/ietf/utils/meetecho.py index 7654f67cd15..943f3789efa 100644 --- a/ietf/utils/meetecho.py +++ b/ietf/utils/meetecho.py @@ -508,8 +508,13 @@ def _should_send_update(self, session): return (timeslot.time - self.slides_notify_time) < now < (timeslot.end_time() + self.slides_notify_time) def add(self, session: "Session", slides: "Document", order: int): + """Add a slide deck to the session + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ if not self._should_send_update(session): - return + return False # Would like to confirm that session.presentations includes the slides Document, but we can't # (same problem regarding unsaved Documents discussed in the docstring) @@ -524,11 +529,16 @@ def add(self, session: "Session", slides: "Document", order: int): "order": order, } ) + return True def delete(self, session: "Session", slides: "Document"): - """Delete a slide deck from the session""" + """Delete a slide deck from the session + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ if not self._should_send_update(session): - return + return False if session.presentations.filter(document=slides).exists(): # "order" problems are very likely to result if we delete slides that are actually still @@ -543,12 +553,17 @@ def delete(self, session: "Session", slides: "Document"): id=slides.pk, ) if session.presentations.filter(document__type_id="slides").exists(): - self.send_update(session) # adjust order to fill in the hole + self._send_update(session) # adjust order to fill in the hole + return True def revise(self, session: "Session", slides: "Document"): - """Replace existing deck with its current state""" + """Replace existing deck with its current state + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ if not self._should_send_update(session): - return + return False sp = session.presentations.filter(document=slides).first() if sp is None: @@ -561,11 +576,13 @@ def revise(self, session: "Session", slides: "Document"): id=slides.pk, ) self.add(session, slides, order) # fill in the hole + return True - def send_update(self, session: "Session"): - if not self._should_send_update(session): - return - + def _send_update(self, session: "Session"): + """Notify of the current state of the session's slides (no time window check) + + This is a private helper - use send_update() (no leading underscore) instead. + """ self.api.update_slide_decks( wg_token=self.wg_token(session.group), session=str(session.pk), @@ -580,3 +597,14 @@ def send_update(self, session: "Session"): for deck in session.presentations.filter(document__type="slides") ] ) + + def send_update(self, session: "Session"): + """Notify of the current state of the session's slides + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ + if not self._should_send_update(session): + return False + self._send_update(session) + return True diff --git a/ietf/utils/tests_meetecho.py b/ietf/utils/tests_meetecho.py index 502e936483a..c076a3df745 100644 --- a/ietf/utils/tests_meetecho.py +++ b/ietf/utils/tests_meetecho.py @@ -547,7 +547,8 @@ def test_add(self, mock_add, mock_wg_token): sm = SlidesManager(settings.MEETECHO_API_CONFIG) session = SessionFactory() slides_doc = DocumentFactory(type_id="slides") - sm.add(session, slides_doc, 13) + retval = sm.add(session, slides_doc, 13) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_add.called) self.assertEqual( @@ -565,6 +566,14 @@ def test_add(self, mock_add, mock_wg_token): ), ) + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.add(session, slides_doc, 14) + self.assertIs(retval, False) + @patch("ietf.utils.meetecho.MeetechoAPI.update_slide_decks") @patch("ietf.utils.meetecho.MeetechoAPI.delete_slide_deck") def test_delete(self, mock_delete, mock_update, mock_wg_token): @@ -580,7 +589,8 @@ def test_delete(self, mock_delete, mock_update, mock_wg_token): sm.delete(session, slides_doc) # can't remove slides still attached to the session self.assertFalse(any([mock_wg_token.called, mock_delete.called, mock_update.called])) - sm.delete(session, removed_slides_doc) + retval = sm.delete(session, removed_slides_doc) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_delete.called) self.assertEqual( @@ -609,9 +619,18 @@ def test_delete(self, mock_delete, mock_update, mock_wg_token): # Delete the other session and check that we don't make the update call slides.delete() - sm.delete(session, slides_doc) + retval = sm.delete(session, slides_doc) + self.assertIs(retval, True) self.assertTrue(mock_delete.called) self.assertFalse(mock_update.called) + + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.delete(session, slides_doc) + self.assertIs(retval, False) @patch("ietf.utils.meetecho.MeetechoAPI.delete_slide_deck") @patch("ietf.utils.meetecho.MeetechoAPI.add_slide_deck") @@ -619,7 +638,8 @@ def test_revise(self, mock_add, mock_delete, mock_wg_token): sm = SlidesManager(settings.MEETECHO_API_CONFIG) slides = SessionPresentationFactory(document__type_id="slides", order=23) slides_doc = slides.document - sm.revise(slides.session, slides.document) + retval = sm.revise(slides.session, slides_doc) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_delete.called) self.assertEqual( @@ -642,13 +662,22 @@ def test_revise(self, mock_add, mock_delete, mock_wg_token): ), ) + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.revise(slides.session, slides_doc) + self.assertIs(retval, False) + @patch("ietf.utils.meetecho.MeetechoAPI.update_slide_decks") def test_send_update(self, mock_send_update, mock_wg_token): sm = SlidesManager(settings.MEETECHO_API_CONFIG) slides = SessionPresentationFactory(document__type_id="slides") SessionPresentationFactory(session=slides.session, document__type_id="agenda") - sm.send_update(slides.session) + retval = sm.send_update(slides.session) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_send_update.called) self.assertEqual( @@ -667,3 +696,11 @@ def test_send_update(self, mock_send_update, mock_wg_token): ] ) ) + + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.send_update(slides.session) + self.assertIs(retval, False) diff --git a/package.json b/package.json index e2e6fd7dab7..fec29275b47 100644 --- a/package.json +++ b/package.json @@ -148,6 +148,7 @@ "ietf/static/js/moment.js", "ietf/static/js/password_strength.js", "ietf/static/js/select2.js", + "ietf/static/js/session_details.js", "ietf/static/js/session_details_form.js", "ietf/static/js/session_form.js", "ietf/static/js/session_request.js", From 619b2aee0f3b4acbc95a44f6fd3e8785163f6a93 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 20 Feb 2026 13:22:49 -0600 Subject: [PATCH 015/188] fix: adjust draft-stream-ietf state descriptions per IESG (#10437) --- ...ge_draft_stream_ietf_state_descriptions.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py diff --git a/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py b/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py new file mode 100644 index 00000000000..c664126da32 --- /dev/null +++ b/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py @@ -0,0 +1,57 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations + + +def forward(apps, schema_editor): + State = apps.get_model("doc", "State") + for name, desc in [ + ( + "Adopted by a WG", + "The individual submission document has been adopted by the Working Group (WG), but some administrative matter still needs to be completed (e.g., a WG document replacing this document with the typical naming convention of 'draft-ietf-wgname-topic-nn' has not yet been submitted).", + ), + ( + "WG Document", + "The document has been identified as a Working Group (WG) document and is under development per Section 7.2 of RFC2418.", + ), + ( + "Waiting for WG Chair Go-Ahead", + "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chairs are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed.", + ), + ( + "Submitted to IESG for Publication", + "The Working Group (WG) document has been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication per Section 7.4 of RFC2418. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", + ), + ]: + State.objects.filter(name=name).update(desc=desc, type="draft-stream-ietf") + + +def reverse(apps, schema_editor): + State = apps.get_model("doc", "State") + for name, desc in [ + ( + "Adopted by a WG", + "The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted.", + ), + ( + "WG Document", + "The document has been adopted by the Working Group (WG) and is under development. A document can only be adopted by one WG at a time. However, a document may be transferred between WGs.", + ), + ( + "Waiting for WG Chair Go-Ahead", + "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed", + ), + ( + "Submitted to IESG for Publication", + "The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", + ), + ]: + State.objects.filter(name=name).update(desc=desc, type="draft-stream-ietf") + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0030_alter_dochistory_title_alter_document_title"), + ] + + operations = [migrations.RunPython(forward, reverse)] From c4be6318f73cbf896b5cc1f3416040e12b4611f4 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 20 Feb 2026 15:25:57 -0400 Subject: [PATCH 016/188] feat: more API fields+filtering; drop RfcAuthor.email field (#10432) * feat: RfcAuthorSerializer.email is current email * refactor: RfcAuthor email field -> property * feat: more RfcMetadataSerializer fields * shepherd email (with a fallback to the draft) * doc ad email * area ad emails * group list email * fix: filter RFCs by any group type * feat: filter by RFC numbers * fix: shepherd -> draft object in response JSON * fix: consistent filter naming * chore: migration * test: update test_notify_rfc_published * fix: RfcAuthor.email() -> Email, not str * fix: update RfcAuthorFactory * fix: consistent blank value in email() * fix: guard against non-prefetched queryset * test: fix nomcom test * refactor: name-addr -> addr for ad/shepherd Also falls back to current primary email for ad/shepherd if the email on record is inactive. --- ietf/api/tests_views_rpc.py | 25 +++++----- ietf/doc/admin.py | 4 +- ietf/doc/api.py | 10 +++- ietf/doc/factories.py | 1 - .../migrations/0031_remove_rfcauthor_email.py | 16 ++++++ ietf/doc/models.py | 6 ++- ietf/doc/serializers.py | 50 +++++++++++++++---- ietf/doc/views_doc.py | 2 +- ietf/group/serializers.py | 32 ++++++++++-- ietf/nomcom/tests.py | 1 - 10 files changed, 115 insertions(+), 32 deletions(-) create mode 100644 ietf/doc/migrations/0031_remove_rfcauthor_email.py diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 09fb40bf6e5..1fbb4c3f023 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -143,22 +143,23 @@ def test_notify_rfc_published(self): self.assertEqual(rfc.title, "RFC " + draft.title) self.assertEqual(rfc.documentauthor_set.count(), 0) self.assertEqual( - list( - rfc.rfcauthor_set.values( - "titlepage_name", - "is_editor", - "person", - "email", - "affiliation", - "country", - ) - ), + [ + { + "titlepage_name": ra.titlepage_name, + "is_editor": ra.is_editor, + "person": ra.person, + "email": ra.email, + "affiliation": ra.affiliation, + "country": ra.country, + } + for ra in rfc.rfcauthor_set.all() + ], [ { "titlepage_name": f"titlepage {author.name}", "is_editor": False, - "person": author.pk, - "email": author.email_address(), + "person": author, + "email": author.email(), "affiliation": "Some Affiliation", "country": "CA", } diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index f082418935a..b604d4f0963 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -242,6 +242,6 @@ def is_deleted(self, instance): class RfcAuthorAdmin(admin.ModelAdmin): list_display = ['id', 'document', 'titlepage_name', 'person', 'email', 'affiliation', 'country', 'order'] - search_fields = ['document__name', 'titlepage_name', 'person__name', 'email__address', 'affiliation', 'country'] - raw_id_fields = ["document", "person", "email"] + search_fields = ['document__name', 'titlepage_name', 'person__name', 'email', 'affiliation', 'country'] + raw_id_fields = ["document", "person"] admin.site.register(RfcAuthor, RfcAuthorAdmin) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 6a4c0c9fd5b..75993f463eb 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -42,13 +42,21 @@ class RfcLimitOffsetPagination(LimitOffsetPagination): max_limit = 500 +class NumberInFilter(filters.BaseInFilter, filters.NumberFilter): + """Filter against a comma-separated list of numbers""" + pass + + class RfcFilter(filters.FilterSet): published = filters.DateFromToRangeFilter() stream = filters.ModelMultipleChoiceFilter( queryset=StreamName.objects.filter(used=True) ) + number = NumberInFilter( + field_name="rfc_number" + ) group = filters.ModelMultipleChoiceFilter( - queryset=Group.objects.wgs(), + queryset=Group.objects.all(), field_name="group__acronym", to_field_name="acronym", ) diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index aad01be04fd..bc387654460 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -391,7 +391,6 @@ class Meta: lambda obj: " ".join([obj.person.initials(), obj.person.last_name()]) ) person = factory.SubFactory('ietf.person.factories.PersonFactory') - email = factory.LazyAttribute(lambda obj: obj.person.email()) affiliation = factory.Faker('company') order = factory.LazyAttribute(lambda o: o.document.rfcauthor_set.count() + 1) diff --git a/ietf/doc/migrations/0031_remove_rfcauthor_email.py b/ietf/doc/migrations/0031_remove_rfcauthor_email.py new file mode 100644 index 00000000000..c4c1911bfe4 --- /dev/null +++ b/ietf/doc/migrations/0031_remove_rfcauthor_email.py @@ -0,0 +1,16 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0030_alter_dochistory_title_alter_document_title"), + ] + + operations = [ + migrations.RemoveField( + model_name="rfcauthor", + name="email", + ), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 8f700bf496b..cc28951be0e 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -937,7 +937,6 @@ class RfcAuthor(models.Model): titlepage_name = models.CharField(max_length=128, blank=False) is_editor = models.BooleanField(default=False) person = ForeignKey(Person, null=True, blank=True, on_delete=models.PROTECT) - email = ForeignKey(Email, help_text="Email address used by author for submission", blank=True, null=True, on_delete=models.PROTECT) affiliation = models.CharField(max_length=100, blank=True, help_text="Organization/company used by author for submission") country = models.CharField(max_length=255, blank=True, help_text="Country used by author for submission") order = models.IntegerField(default=1) @@ -951,6 +950,11 @@ class Meta: models.Index(fields=["document", "order"]) ] + @property + def email(self) -> Email | None: + return self.person.email() if self.person else None + + class DocumentAuthorInfo(models.Model): person = ForeignKey(Person) # email should only be null for some historic documents diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index e8d373164ba..b054b074d71 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -9,14 +9,20 @@ from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from ietf.group.serializers import GroupSerializer +from ietf.group.serializers import ( + AreaDirectorSerializer, + AreaSerializer, + GroupSerializer, +) from ietf.name.serializers import StreamNameSerializer +from ietf.utils import log from .models import Document, DocumentAuthor, RfcAuthor class RfcAuthorSerializer(serializers.ModelSerializer): """Serializer for an RfcAuthor / DocumentAuthor in a response""" + email = serializers.EmailField(source="email.address", read_only=True) datatracker_person_path = serializers.URLField( source="person.get_absolute_url", required=False, @@ -29,7 +35,7 @@ class Meta: "titlepage_name", "is_editor", "person", - "email", # relies on email.pk being email.address + "email", "affiliation", "country", "datatracker_person_path", @@ -48,7 +54,6 @@ def to_representation(self, instance): titlepage_name=document_author.person.plain_name(), is_editor=False, person=document_author.person, - email=document_author.email, affiliation=document_author.affiliation, country=document_author.country, order=document_author.order, @@ -174,10 +179,16 @@ def to_representation(self, instance: Document): return super().to_representation(instance=RfcStatus.from_document(instance)) +class ShepherdSerializer(serializers.Serializer): + email = serializers.EmailField(source="email_address") + + class RelatedDraftSerializer(serializers.Serializer): id = serializers.IntegerField(source="source.id") name = serializers.CharField(source="source.name") title = serializers.CharField(source="source.title") + shepherd = ShepherdSerializer(source="source.shepherd") + ad = AreaDirectorSerializer(source="source.ad") class RelatedRfcSerializer(serializers.Serializer): @@ -205,15 +216,23 @@ class RfcFormatSerializer(serializers.Serializer): class RfcMetadataSerializer(serializers.ModelSerializer): - """Serialize metadata of an RFC""" + """Serialize metadata of an RFC + + This needs to be called with a Document queryset that has been processed with + api.augment_rfc_queryset() or it very likely will not work. Some of the typing + refers to Document, but this should really be WithAnnotations[Document, ...]. + However, have not been able to make that work yet. + """ number = serializers.IntegerField(source="rfc_number") published = serializers.DateField() status = RfcStatusSerializer(source="*") authors = serializers.SerializerMethodField() group = GroupSerializer() - area = GroupSerializer(source="group.area", required=False) + area = AreaSerializer(source="group.area", required=False) stream = StreamNameSerializer() + ad = AreaDirectorSerializer(read_only=True) + group_list_email = serializers.EmailField(source="group.list_email", read_only=True) identifiers = serializers.SerializerMethodField() draft = serializers.SerializerMethodField() obsoletes = RelatedRfcSerializer(many=True, read_only=True) @@ -239,6 +258,8 @@ class Meta: "group", "area", "stream", + "ad", + "group_list_email", "identifiers", "obsoletes", "obsoleted_by", @@ -276,11 +297,20 @@ def get_identifiers(self, doc: Document): return DocIdentifierSerializer(instance=identifiers, many=True).data @extend_schema_field(RelatedDraftSerializer) - def get_draft(self, object): - try: - related_doc = object.drafts[0] - except IndexError: - return None + def get_draft(self, doc: Document): + if hasattr(doc, "drafts"): + # This is the expected case - drafts is added by a Prefetch in + # the augment_rfc_queryset() method. + try: + related_doc = doc.drafts[0] + except IndexError: + return None + else: + # Fallback in case augment_rfc_queryset() was not called + log.log( + f"Warning: {self.__class__}.get_draft() called without prefetched draft" + ) + related_doc = doc.came_from_draft() return RelatedDraftSerializer(related_doc).data diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 0578da1b772..0ae7520681d 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -1657,7 +1657,7 @@ def extract_name(s): doc.rfcauthor_set if doc.type_id == "rfc" and doc.rfcauthor_set.exists() else doc.documentauthor_set - ).select_related("person", "email").order_by("order") + ).select_related("person").prefetch_related("person__email_set").order_by("order") data["authors"] = [ { "name": author.titlepage_name if hasattr(author, "titlepage_name") else author.person.name, diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py index 08e6bba81af..85f209019c3 100644 --- a/ietf/group/serializers.py +++ b/ietf/group/serializers.py @@ -1,11 +1,37 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved """django-rest-framework serializers""" + +from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from .models import Group +from ietf.person.models import Email +from .models import Group, Role class GroupSerializer(serializers.ModelSerializer): class Meta: model = Group - fields = ["acronym", "name", "type"] + fields = ["acronym", "name", "type", "list_email"] + + +class AreaDirectorSerializer(serializers.Serializer): + """Serialize an area director + + Works with Email or Role + """ + + email = serializers.SerializerMethodField() + + @extend_schema_field(serializers.EmailField) + def get_email(self, instance: Email | Role): + if isinstance(instance, Role): + return instance.email.email_address() + return instance.email_address() + + +class AreaSerializer(serializers.ModelSerializer): + ads = AreaDirectorSerializer(many=True, read_only=True) + + class Meta: + model = Group + fields = ["acronym", "name", "type", "ads"] diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index b6e8c57da72..210788ce079 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -2528,7 +2528,6 @@ def test_get_qualified_author_queryset(self): document=rfc, person=people[0], titlepage_name="P. Zero", - email=people[0].email_set.first(), ) self.assertCountEqual( get_qualified_author_queryset(base_qs, now - 5 * one_year, now), From d7319030f3d3a38bf0c048e713bc1f068ca228ed Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 20 Feb 2026 16:12:43 -0400 Subject: [PATCH 017/188] chore: renumber migrations (#10441) --- ...remove_rfcauthor_email.py => 0032_remove_rfcauthor_email.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename ietf/doc/migrations/{0031_remove_rfcauthor_email.py => 0032_remove_rfcauthor_email.py} (80%) diff --git a/ietf/doc/migrations/0031_remove_rfcauthor_email.py b/ietf/doc/migrations/0032_remove_rfcauthor_email.py similarity index 80% rename from ietf/doc/migrations/0031_remove_rfcauthor_email.py rename to ietf/doc/migrations/0032_remove_rfcauthor_email.py index c4c1911bfe4..a0e147da591 100644 --- a/ietf/doc/migrations/0031_remove_rfcauthor_email.py +++ b/ietf/doc/migrations/0032_remove_rfcauthor_email.py @@ -5,7 +5,7 @@ class Migration(migrations.Migration): dependencies = [ - ("doc", "0030_alter_dochistory_title_alter_document_title"), + ("doc", "0031_change_draft_stream_ietf_state_descriptions"), ] operations = [ From abf5e0d97ca38ede129b218d3a895f9bb5ab441b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 20 Feb 2026 17:45:31 -0400 Subject: [PATCH 018/188] fix: allow null for shepherd/ad (#10443) --- ietf/doc/serializers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index b054b074d71..e42a6a0293b 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -187,8 +187,8 @@ class RelatedDraftSerializer(serializers.Serializer): id = serializers.IntegerField(source="source.id") name = serializers.CharField(source="source.name") title = serializers.CharField(source="source.title") - shepherd = ShepherdSerializer(source="source.shepherd") - ad = AreaDirectorSerializer(source="source.ad") + shepherd = ShepherdSerializer(source="source.shepherd", allow_null=True) + ad = AreaDirectorSerializer(source="source.ad", allow_null=True) class RelatedRfcSerializer(serializers.Serializer): @@ -231,7 +231,7 @@ class RfcMetadataSerializer(serializers.ModelSerializer): group = GroupSerializer() area = AreaSerializer(source="group.area", required=False) stream = StreamNameSerializer() - ad = AreaDirectorSerializer(read_only=True) + ad = AreaDirectorSerializer(read_only=True, allow_null=True) group_list_email = serializers.EmailField(source="group.list_email", read_only=True) identifiers = serializers.SerializerMethodField() draft = serializers.SerializerMethodField() From 18902ff1be5746cafc958561e563f00be5f05176 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 24 Feb 2026 13:10:42 -0400 Subject: [PATCH 019/188] fix: more accurate group areas (red API) (#10462) * fix: RFC area only for ietf stream * fix: no ADs for inactive areas --- ietf/doc/serializers.py | 19 ++++++++++++++++++- ietf/group/serializers.py | 9 ++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index e42a6a0293b..36076c30be3 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -229,7 +229,7 @@ class RfcMetadataSerializer(serializers.ModelSerializer): status = RfcStatusSerializer(source="*") authors = serializers.SerializerMethodField() group = GroupSerializer() - area = AreaSerializer(source="group.area", required=False) + area = serializers.SerializerMethodField() stream = StreamNameSerializer() ad = AreaDirectorSerializer(read_only=True, allow_null=True) group_list_email = serializers.EmailField(source="group.list_email", read_only=True) @@ -287,6 +287,23 @@ def get_authors(self, doc: Document): many=True, ).data + @extend_schema_field(AreaSerializer(required=False)) + def get_area(self, doc: Document): + """Get area for the RFC + + This logic might be better moved to Document or a combination of Document + and Group. The current (2026-02-24) Group.area() method is not strict enough: + it does not limit to WG groups or IETF-stream documents. + """ + if doc.stream_id != "ietf": + return None + if doc.group is None: + return None + parent = doc.group.parent + if parent.type_id == "area": + return AreaSerializer(parent).data + return None + @extend_schema_field(DocIdentifierSerializer(many=True)) def get_identifiers(self, doc: Document): identifiers = [] diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py index 85f209019c3..4ba92232c5b 100644 --- a/ietf/group/serializers.py +++ b/ietf/group/serializers.py @@ -30,8 +30,15 @@ def get_email(self, instance: Email | Role): class AreaSerializer(serializers.ModelSerializer): - ads = AreaDirectorSerializer(many=True, read_only=True) + ads = serializers.SerializerMethodField() class Meta: model = Group fields = ["acronym", "name", "type", "ads"] + + @extend_schema_field(AreaDirectorSerializer(many=True)) + def get_ads(self, area: Group): + return AreaDirectorSerializer( + area.ads() if area.is_active else Role.objects.none(), + many=True, + ).data From 07efd2b078a461da2eb7e197fc91f2ae0b45ac40 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 25 Feb 2026 12:58:57 -0400 Subject: [PATCH 020/188] fix: unbreak red API + group serializer tests (#10467) * test: group serializer tests * fix: Group.ads is a property * fix: no need for type in AreaSerializer --- ietf/group/serializers.py | 4 +- ietf/group/tests_serializers.py | 90 +++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 ietf/group/tests_serializers.py diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py index 4ba92232c5b..db3b37af48a 100644 --- a/ietf/group/serializers.py +++ b/ietf/group/serializers.py @@ -34,11 +34,11 @@ class AreaSerializer(serializers.ModelSerializer): class Meta: model = Group - fields = ["acronym", "name", "type", "ads"] + fields = ["acronym", "name", "ads"] @extend_schema_field(AreaDirectorSerializer(many=True)) def get_ads(self, area: Group): return AreaDirectorSerializer( - area.ads() if area.is_active else Role.objects.none(), + area.ads if area.is_active else Role.objects.none(), many=True, ).data diff --git a/ietf/group/tests_serializers.py b/ietf/group/tests_serializers.py new file mode 100644 index 00000000000..bf29e6c8fd2 --- /dev/null +++ b/ietf/group/tests_serializers.py @@ -0,0 +1,90 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from ietf.group.factories import RoleFactory, GroupFactory +from ietf.group.serializers import ( + AreaDirectorSerializer, + AreaSerializer, + GroupSerializer, +) +from ietf.person.factories import EmailFactory +from ietf.utils.test_utils import TestCase + + +class GroupSerializerTests(TestCase): + def test_serializes(self): + wg = GroupFactory() + serialized = GroupSerializer(wg).data + self.assertEqual( + serialized, + { + "acronym": wg.acronym, + "name": wg.name, + "type": "wg", + "list_email": wg.list_email, + }, + ) + + +class AreaDirectorSerializerTests(TestCase): + def test_serializes_role(self): + """Should serialize a Role correctly""" + role = RoleFactory(group__type_id="area", name_id="ad") + serialized = AreaDirectorSerializer(role).data + self.assertEqual( + serialized, + {"email": role.email.email_address()}, + ) + + def test_serializes_email(self): + """Should serialize an Email correctly""" + email = EmailFactory() + serialized = AreaDirectorSerializer(email).data + self.assertEqual( + serialized, + {"email": email.email_address()}, + ) + + +class AreaSerializerTests(TestCase): + def test_serializes_active_area(self): + """Should serialize an active area correctly""" + area = GroupFactory(type_id="area", state_id="active") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) + ad_roles = RoleFactory.create_batch(2, group=area, name_id="ad") + serialized = AreaSerializer(area).data + self.assertEqual(serialized["acronym"], area.acronym) + self.assertEqual(serialized["name"], area.name) + self.assertCountEqual( + serialized["ads"], + [{"email": ad.email.email_address()} for ad in ad_roles], + ) + + def test_serializes_inactive_area(self): + """Should serialize an inactive area correctly""" + area = GroupFactory(type_id="area", state_id="conclude") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) + RoleFactory.create_batch(2, group=area, name_id="ad") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) From b81249884877e20c6e311478fe25b472c869c555 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 25 Feb 2026 13:24:30 -0600 Subject: [PATCH 021/188] feat: notify errata system of new rfc (#10465) * feat: notify errata system of new rfc * chore: ruff --- docker/configs/settings_local.py | 4 ++++ ietf/api/tests_views_rpc.py | 28 +++++++++++++++++++------ ietf/api/views_rpc.py | 9 +++++++- ietf/doc/tasks.py | 5 +++++ ietf/doc/utils_errata.py | 35 ++++++++++++++++++++++++++++++++ ietf/settings.py | 5 +++++ 6 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 ietf/doc/utils_errata.py diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index e357ce3f735..1d4e6916b94 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -105,3 +105,7 @@ "ietf.api.red_api" : ["devtoken", "redtoken"], # Not a real secret "ietf.api.views_rpc" : ["devtoken"], # Not a real secret } + +# Errata system api configuration +ERRATA_METADATA_NOTIFICATION_URL = "http://host.docker.internal:8808/api/rfc_metadata_update/" +ERRATA_METADATA_NOTIFICATION_API_KEY = "not a real secret" diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 1fbb4c3f023..6a5a5c9b88e 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -9,9 +9,10 @@ from django.db.models.functions import Coalesce from django.test.utils import override_settings from django.urls import reverse as urlreverse +import mock from ietf.blobdb.models import Blob -from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory +from ietf.doc.factories import IndividualDraftFactory, RfcFactory, WgDraftFactory, WgRfcFactory from ietf.doc.models import RelatedDocument, Document from ietf.group.factories import RoleFactory, GroupFactory from ietf.person.factories import PersonFactory @@ -77,7 +78,8 @@ def test_draftviewset_references(self): self.assertEqual(refs[0]["name"], draft_bar.name) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) - def test_notify_rfc_published(self): + @mock.patch("ietf.doc.tasks.signal_update_rfc_metadata_task.delay") + def test_notify_rfc_published(self, mock_task_delay): url = urlreverse("ietf.api.purple_api.notify_rfc_published") area = GroupFactory(type_id="area") rfc_group = GroupFactory(type_id="wg") @@ -90,6 +92,8 @@ def test_notify_rfc_published(self): ) rfc_stream_id = "ise" assert isinstance(draft, Document), "WgDraftFactory should generate a Document" + updates = RfcFactory.create_batch(2) + obsoletes = RfcFactory.create_batch(2) unused_rfc_number = ( Document.objects.filter(rfc_number__isnull=False).aggregate( unused_rfc_number=Max("rfc_number") + 1 @@ -120,8 +124,8 @@ def test_notify_rfc_published(self): "pages": draft.pages + 10, "std_level": "ps", "ad": rfc_ad.pk, - "obsoletes": [], - "updates": [], + "obsoletes": [o.rfc_number for o in obsoletes], + "updates": [o.rfc_number for o in updates], "subseries": [], } r = self.client.post(url, data=post_data, format="json") @@ -172,13 +176,25 @@ def test_notify_rfc_published(self): self.assertEqual(rfc.pages, draft.pages + 10) self.assertEqual(rfc.std_level_id, "ps") self.assertEqual(rfc.ad, rfc_ad) - self.assertEqual(rfc.related_that_doc("obs"), []) - self.assertEqual(rfc.related_that_doc("updates"), []) + self.assertEqual(set(rfc.related_that_doc("obs")), set([o for o in obsoletes])) + self.assertEqual( + set(rfc.related_that_doc("updates")), set([o for o in updates]) + ) self.assertEqual(rfc.part_of(), []) self.assertEqual(draft.get_state().slug, "rfc") # todo test non-empty relationships # todo test references (when updating that is part of the handling) + self.assertTrue(mock_task_delay.called) + mock_args, mock_kwargs = mock_task_delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_rfc_number_list = [rfc.rfc_number] + expected_rfc_number_list.extend( + [d.rfc_number for d in updates + obsoletes] + ) + expected_rfc_number_list = sorted(set(expected_rfc_number_list)) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) def test_upload_rfc_files(self): def _valid_post_data(): diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 2bf16480f2b..9273590b283 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -38,6 +38,7 @@ from ietf.doc.models import Document, DocHistory, RfcAuthor from ietf.doc.serializers import RfcAuthorSerializer from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage +from ietf.doc.tasks import signal_update_rfc_metadata_task from ietf.person.models import Email, Person @@ -362,7 +363,7 @@ def post(self, request): serializer.is_valid(raise_exception=True) # Create RFC try: - serializer.save() + rfc = serializer.save() except IntegrityError as err: if Document.objects.filter( rfc_number=serializer.validated_data["rfc_number"] @@ -375,6 +376,12 @@ def post(self, request): f"Unable to publish: {err}", code="unknown-integrity-error", ) + rfc_number_list = [rfc.rfc_number] + rfc_number_list.extend( + [d.rfc_number for d in rfc.related_that_doc(("updates", "obs"))] + ) + rfc_number_list = sorted(set(rfc_number_list)) + signal_update_rfc_metadata_task.delay(rfc_number_list=rfc_number_list) return Response(NotificationAckSerializer().data) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index b463b9cecf2..90f4c80af57 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -35,6 +35,7 @@ investigate_fragment, ) from .utils_bofreq import fixup_bofreq_timestamps +from .utils_errata import signal_update_rfc_metadata @shared_task @@ -155,3 +156,7 @@ def rebuild_reference_relations_task(doc_names: list[str]): @shared_task def fixup_bofreq_timestamps_task(): # pragma: nocover fixup_bofreq_timestamps() + +@shared_task +def signal_update_rfc_metadata_task(rfc_number_list=()): + signal_update_rfc_metadata(rfc_number_list) diff --git a/ietf/doc/utils_errata.py b/ietf/doc/utils_errata.py new file mode 100644 index 00000000000..539262151ff --- /dev/null +++ b/ietf/doc/utils_errata.py @@ -0,0 +1,35 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import requests + +from django.conf import settings + +from ietf.utils.log import log + + +def signal_update_rfc_metadata(rfc_number_list=()): + key = getattr(settings, "ERRATA_METADATA_NOTIFICATION_API_KEY", None) + if key is not None: + headers = {"X-Api-Key": settings.ERRATA_METADATA_NOTIFICATION_API_KEY} + post_dict = { + "rfc_number_list": list(rfc_number_list), + } + try: + response = requests.post( + settings.ERRATA_METADATA_NOTIFICATION_URL, + headers=headers, + json=post_dict, + timeout=settings.DEFAULT_REQUESTS_TIMEOUT, + ) + except requests.Timeout as e: + log( + f"POST request timed out for {settings.ERRATA_METADATA_NOTIFICATION_URL} ]: {e}" + ) + # raise RuntimeError(f'POST request timed out for {settings.ERRATA_METADATA_NOTIFICATION_URL}') from e + return + if response.status_code != 200: + log( + f"POST request failed for {settings.ERRATA_METADATA_NOTIFICATION_URL} ]: {response.status_code} {response.text}" + ) + else: + log("No API key configured for errata metadata notification, skipping") diff --git a/ietf/settings.py b/ietf/settings.py index 565e8825a9f..71b110d762a 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1368,6 +1368,11 @@ def skip_unreadable_post(record): MEETECHO_AUDIO_STREAM_URL = "https://mp3.conf.meetecho.com/ietf{session.meeting.number}/{session.pk}.m3u" MEETECHO_SESSION_RECORDING_URL = "https://meetecho-player.ietf.org/playout/?session={session_label}" +# Errata system api configuration +# settings should provide +# ERRATA_METADATA_NOTIFICATION_URL +# ERRATA_METADATA_NOTIFICATION_API_KEY + # Put the production SECRET_KEY in settings_local.py, and also any other # sensitive or site-specific changes. DO NOT commit settings_local.py to svn. from ietf.settings_local import * # pyflakes:ignore pylint: disable=wildcard-import From da5614c4963c3dc4ff8e901c1edd888296219a0d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 25 Feb 2026 18:14:47 -0400 Subject: [PATCH 022/188] test: avoid random fail in test_rfc_index (#10469) --- ietf/sync/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index bcc87a43aab..21d6cb5cd5c 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -301,6 +301,7 @@ def test_rfc_index(self): ad=Person.objects.get(user__username='ad'), external_url="http://my-external-url.example.com", note="this is a note", + pages=54, # make sure this is not 42 ) DocumentAuthorFactory.create_batch(2, document=draft_doc) draft_doc.action_holders.add(draft_doc.ad) # not normally set, but add to be sure it's cleared From c1c24d012d23135725f0206dbe1a6be1e2a7fef4 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 27 Feb 2026 14:31:40 -0400 Subject: [PATCH 023/188] feat: RFC metadata update API (#10476) * feat: more editable RFC fields for API (WIP) Checkpoint commit! * chore: avoid requiring prefetch Makes some fields write-only to achieve this. * refactor: replace EditableRfcSerializer * fix: mark read-only field properly * refactor: SubseriesNameField * test: EditableRfcSerializer * refactor: DocEvent adjustment * feat: record person ids for authors * chore: adjust history message * fix: always save!! * fix: better msg formatting * fix: _almost_ always save!! * fix: lint * refactor: rename var --- ietf/api/serializers_rpc.py | 227 ++++++++++++++++++++++++++---- ietf/api/tests_serializers_rpc.py | 139 ++++++++++++++++++ ietf/api/views_rpc.py | 12 +- ietf/doc/serializers.py | 1 + ietf/doc/utils.py | 16 ++- 5 files changed, 361 insertions(+), 34 deletions(-) create mode 100644 ietf/api/tests_serializers_rpc.py diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 34e2c791c09..d5f5363990a 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -216,32 +216,24 @@ class Meta: read_only_fields = ["id", "name"] -class EditableRfcSerializer(serializers.ModelSerializer): - # Would be nice to reconcile this with ietf.doc.serializers.RfcSerializer. - # The purposes of that serializer (representing data for Red) and this one - # (accepting updates from Purple) are different enough that separate formats - # may be needed, but if not it'd be nice to have a single RfcSerializer that - # can serve both. - # - # For now, only handles authors - authors = RfcAuthorSerializer(many=True, min_length=1, source="rfcauthor_set") +def _update_authors(rfc, authors_data): + # Construct unsaved instances from validated author data + new_authors = [RfcAuthor(**authdata) for authdata in authors_data] + # Update the RFC with the new author set + with transaction.atomic(): + change_events = update_rfcauthors(rfc, new_authors) + for event in change_events: + event.save() + return change_events - class Meta: - model = Document - fields = ["id", "authors"] - def update(self, instance, validated_data): - assert isinstance(instance, Document) - authors_data = validated_data.pop("rfcauthor_set", None) - if authors_data is not None: - # Construct unsaved instances from validated author data - new_authors = [RfcAuthor(**ad) for ad in authors_data] - # Update the RFC with the new author set - with transaction.atomic(): - change_events = update_rfcauthors(instance, new_authors) - for event in change_events: - event.save() - return instance +class SubseriesNameField(serializers.RegexField): + + def __init__(self, **kwargs): + # pattern: no leading 0, finite length (arbitrarily set to 5 digits) + regex = r"^(bcp|std|fyi)[1-9][0-9]{0,4}$" + super().__init__(regex, **kwargs) + class RfcPubSerializer(serializers.ModelSerializer): @@ -283,13 +275,7 @@ class RfcPubSerializer(serializers.ModelSerializer): slug_field="rfc_number", queryset=Document.objects.filter(type_id="rfc"), ) - subseries = serializers.ListField( - child=serializers.RegexField( - required=False, - # pattern: no leading 0, finite length (arbitrarily set to 5 digits) - regex=r"^(bcp|std|fyi)[1-9][0-9]{0,4}$", - ) - ) + subseries = serializers.ListField(child=SubseriesNameField(required=False)) # N.b., authors is _not_ a field on Document! authors = RfcAuthorSerializer(many=True) @@ -327,6 +313,9 @@ def validate(self, data): ) return data + def update(self, instance, validated_data): + raise RuntimeError("Cannot update with this serializer") + def create(self, validated_data): """Publish an RFC""" published = validated_data.pop("published") @@ -515,6 +504,182 @@ def _create_rfc(self, validated_data): return rfc +class EditableRfcSerializer(serializers.ModelSerializer): + # Would be nice to reconcile this with ietf.doc.serializers.RfcSerializer. + # The purposes of that serializer (representing data for Red) and this one + # (accepting updates from Purple) are different enough that separate formats + # may be needed, but if not it'd be nice to have a single RfcSerializer that + # can serve both. + # + # Should also consider whether this and RfcPubSerializer should merge. + # + # Treats published and subseries fields as write-only. This isn't quite correct, + # but makes it easier and we don't currently use the serialized value except for + # debugging. + published = serializers.DateTimeField( + default_timezone=datetime.timezone.utc, + write_only=True, + ) + authors = RfcAuthorSerializer(many=True, min_length=1, source="rfcauthor_set") + subseries = serializers.ListField( + child=SubseriesNameField(required=False), + write_only=True, + ) + + class Meta: + model = Document + fields = [ + "published", + "title", + "authors", + "stream", + "abstract", + "pages", + "std_level", + "subseries", + ] + + def create(self, validated_data): + raise RuntimeError("Cannot create with this serializer") + + def update(self, instance, validated_data): + assert isinstance(instance, Document) + assert instance.type_id == "rfc" + rfc = instance # get better name + + system_person = Person.objects.get(name="(System)") + + # Remove data that needs special handling. Use a singleton object to detect + # missing values in case we ever support a value that needs None as an option. + omitted = object() + published = validated_data.pop("published", omitted) + subseries = validated_data.pop("subseries", omitted) + authors_data = validated_data.pop("rfcauthor_set", omitted) + + # Transaction to clean up if something fails + with transaction.atomic(): + # update the rfc Document itself + rfc_changes = [] + rfc_events = [] + + for attr, new_value in validated_data.items(): + old_value = getattr(rfc, attr) + if new_value != old_value: + rfc_changes.append( + f"changed {attr} to '{new_value}' from '{old_value}'" + ) + setattr(rfc, attr, new_value) + if len(rfc_changes) > 0: + rfc_change_summary = f"{', '.join(rfc_changes)}" + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + by=system_person, + type="sync_from_rfc_editor", + desc=f"Changed metadata: {rfc_change_summary}", + ) + ) + if authors_data is not omitted: + rfc_events.extend(_update_authors(instance, authors_data)) + + if published is not omitted: + published_event = rfc.latest_event(type="published_rfc") + if published_event is None: + # unexpected, but possible in theory + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="published_rfc", + time=published, + by=system_person, + desc="RFC published", + ) + ) + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="sync_from_rfc_editor", + by=system_person, + desc=( + f"Set publication timestamp to {published.isoformat()}" + ), + ) + ) + else: + original_pub_time = published_event.time + if published != original_pub_time: + published_event.time = published + published_event.save() + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="sync_from_rfc_editor", + by=system_person, + desc=( + f"Changed publication time to " + f"{published.isoformat()} from " + f"{original_pub_time.isoformat()}" + ) + ) + ) + + # update subseries relations + if subseries is not omitted: + for subseries_doc_name in subseries: + ss_slug = subseries_doc_name[:3] + subseries_doc, ss_doc_created = Document.objects.get_or_create( + type_id=ss_slug, name=subseries_doc_name + ) + if ss_doc_created: + subseries_doc.docevent_set.create( + type=f"{ss_slug}_doc_created", + by=system_person, + desc=f"Created {subseries_doc_name} via update of {rfc.name}", + ) + _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create( + relationship_id="contains", target=rfc + ) + if ss_rel_created: + subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + rfc_events.append( + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + ) + # Delete subseries relations that are no longer current + stale_subseries_relations = rfc.relations_that("contains").exclude( + source__name__in=subseries + ) + for stale_relation in stale_subseries_relations: + stale_subseries_doc = stale_relation.source + rfc_events.append( + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Removed {rfc.name} from {stale_subseries_doc.name}", + ) + ) + stale_subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Removed {rfc.name} from {stale_subseries_doc.name}", + ) + stale_subseries_relations.delete() + if len(rfc_events) > 0: + rfc.save_with_history(rfc_events) + return rfc + + class RfcFileSerializer(serializers.Serializer): # The structure of this serializer is constrained by what openapi-generator-cli's # python generator can correctly serialize as multipart/form-data. It does not diff --git a/ietf/api/tests_serializers_rpc.py b/ietf/api/tests_serializers_rpc.py new file mode 100644 index 00000000000..1babb4c30f9 --- /dev/null +++ b/ietf/api/tests_serializers_rpc.py @@ -0,0 +1,139 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from django.utils import timezone + +from ietf.utils.test_utils import TestCase +from ietf.doc.models import Document +from ietf.doc.factories import WgRfcFactory +from .serializers_rpc import EditableRfcSerializer + + +class EditableRfcSerializerTests(TestCase): + def test_create(self): + serializer = EditableRfcSerializer( + data={ + "published": timezone.now(), + "title": "Yadda yadda yadda", + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + "stream": "ietf", + "abstract": "A long time ago in a galaxy far, far away...", + "pages": 3, + "std_level": "inf", + "subseries": ["fyi999"], + } + ) + self.assertTrue(serializer.is_valid()) + with self.assertRaises(RuntimeError, msg="serializer does not allow create()"): + serializer.save() + + def test_update(self): + rfc = WgRfcFactory(pages=10) + serializer = EditableRfcSerializer( + instance=rfc, + data={ + "published": timezone.now(), + "title": "Yadda yadda yadda", + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + "stream": "ise", + "abstract": "A long time ago in a galaxy far, far away...", + "pages": 3, + "std_level": "inf", + "subseries": ["fyi999"], + }, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(result.title, "Yadda yadda yadda") + self.assertEqual( + list( + result.rfcauthor_set.values( + "titlepage_name", "is_editor", "affiliation", "country" + ) + ), + [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + ) + self.assertEqual(result.stream_id, "ise") + self.assertEqual( + result.abstract, "A long time ago in a galaxy far, far away..." + ) + self.assertEqual(result.pages, 3) + self.assertEqual(result.std_level_id, "inf") + self.assertEqual( + result.part_of(), + [Document.objects.get(name="fyi999")], + ) + + def test_partial_update(self): + # We could test other permutations of fields, but authors is a partial update + # we know we are going to use, so verifying that one in particular. + rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan") + serializer = EditableRfcSerializer( + partial=True, + instance=rfc, + data={ + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + }, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(rfc.title, "padawan") + self.assertEqual( + list( + result.rfcauthor_set.values( + "titlepage_name", "is_editor", "affiliation", "country" + ) + ), + [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + ) + self.assertEqual(result.stream_id, "ietf") + self.assertEqual(result.abstract, "do or do not") + self.assertEqual(result.pages, 10) + self.assertEqual(result.std_level_id, "ps") + self.assertEqual(result.part_of(), []) + + # Test only a field on the Document itself to be sure that it works + serializer = EditableRfcSerializer( + partial=True, + instance=rfc, + data={"title": "jedi master"}, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(rfc.title, "jedi master") diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 9273590b283..8862bbf8660 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -35,7 +35,7 @@ NotificationAckSerializer, RfcPubSerializer, RfcFileSerializer, EditableRfcSerializer, ) -from ietf.doc.models import Document, DocHistory, RfcAuthor +from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent from ietf.doc.serializers import RfcAuthorSerializer from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage from ietf.doc.tasks import signal_update_rfc_metadata_task @@ -279,6 +279,16 @@ class RfcViewSet(mixins.UpdateModelMixin, viewsets.GenericViewSet): lookup_field = "rfc_number" serializer_class = EditableRfcSerializer + def perform_update(self, serializer): + DocEvent.objects.create( + doc=serializer.instance, + rev=serializer.instance.rev, + by=Person.objects.get(name="(System)"), + type="sync_from_rfc_editor", + desc="Metadata update from RFC Editor", + ) + super().perform_update(serializer) + @action(detail=False, serializer_class=OriginalStreamSerializer) def rfc_original_stream(self, request): rfcs = self.get_queryset().annotate( diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index 36076c30be3..a7ea640be81 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -27,6 +27,7 @@ class RfcAuthorSerializer(serializers.ModelSerializer): source="person.get_absolute_url", required=False, help_text="URL for person link (relative to datatracker base URL)", + read_only=True, ) class Meta: diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 42fab7d4724..396b3fcfa47 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -740,14 +740,26 @@ def _rfcauthor_from_documentauthor(docauthor: DocumentAuthor) -> RfcAuthor: new_author.document = rfc new_author.order = order + 1 new_author.save() - changes.append(f'Added "{new_author.titlepage_name}" as author') + if new_author.person_id is not None: + person_desc = f"Person {new_author.person_id}" + else: + person_desc = "no Person linked" + changes.append( + f'Added "{new_author.titlepage_name}" ({person_desc}) as author' + ) # Any authors left in original_authors are no longer in the list, so remove them for removed_author in original_authors: # Skip actual removal of old authors if we are converting from the # DocumentAuthor models - the original_authors were just stand-ins anyway. if not converting_from_docauthors: removed_author.delete() - changes.append(f'Removed "{removed_author.titlepage_name}" as author') + if removed_author.person_id is not None: + person_desc = f"Person {removed_author.person_id}" + else: + person_desc = "no Person linked" + changes.append( + f'Removed "{removed_author.titlepage_name}" ({person_desc}) as author' + ) # Create DocEvents, but leave it up to caller to save if by is None: by = Person.objects.get(name="(System)") From 481054511b9f07a47c41f854105e00616e61d3e2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 3 Mar 2026 11:36:45 -0400 Subject: [PATCH 024/188] feat: add area to FullDraftSerializer (#10487) * refactor: Document.area() + serializer * feat: add area to FullDraftSerializer --- ietf/api/serializers_rpc.py | 3 +++ ietf/doc/models.py | 16 ++++++++++++++++ ietf/doc/serializers.py | 19 +------------------ 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index d5f5363990a..e51b917be4a 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -27,6 +27,7 @@ update_rfcauthors, ) from ietf.group.models import Group +from ietf.group.serializers import AreaSerializer from ietf.name.models import StreamName, StdLevelName from ietf.person.models import Person from ietf.utils import log @@ -115,6 +116,7 @@ class FullDraftSerializer(serializers.ModelSerializer): name = serializers.CharField(max_length=255) title = serializers.CharField(max_length=255) group = serializers.SlugRelatedField(slug_field="acronym", read_only=True) + area = AreaSerializer(read_only=True) # Other fields we need to add / adjust source_format = serializers.SerializerMethodField() @@ -133,6 +135,7 @@ class Meta: "stream", "title", "group", + "area", "abstract", "pages", "source_format", diff --git a/ietf/doc/models.py b/ietf/doc/models.py index cc28951be0e..f1b319367e5 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1147,6 +1147,22 @@ def request_closed_time(self, review_req): e = self.latest_event(ReviewRequestDocEvent, type="closed_review_request", review_request=review_req) return e.time if e and e.time else None + @property + def area(self) -> Group | None: + """Get area for document, if one exists + + None for non-IETF-stream documents. N.b., this is stricter than Group.area() and + uses different logic from Document.area_acronym(). + """ + if self.stream_id != "ietf": + return None + if self.group is None: + return None + parent = self.group.parent + if parent.type_id == "area": + return parent + return None + def area_acronym(self): g = self.group if g: diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index a7ea640be81..139ae9aa7e6 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -230,7 +230,7 @@ class RfcMetadataSerializer(serializers.ModelSerializer): status = RfcStatusSerializer(source="*") authors = serializers.SerializerMethodField() group = GroupSerializer() - area = serializers.SerializerMethodField() + area = AreaSerializer(read_only=True) stream = StreamNameSerializer() ad = AreaDirectorSerializer(read_only=True, allow_null=True) group_list_email = serializers.EmailField(source="group.list_email", read_only=True) @@ -288,23 +288,6 @@ def get_authors(self, doc: Document): many=True, ).data - @extend_schema_field(AreaSerializer(required=False)) - def get_area(self, doc: Document): - """Get area for the RFC - - This logic might be better moved to Document or a combination of Document - and Group. The current (2026-02-24) Group.area() method is not strict enough: - it does not limit to WG groups or IETF-stream documents. - """ - if doc.stream_id != "ietf": - return None - if doc.group is None: - return None - parent = doc.group.parent - if parent.type_id == "area": - return AreaSerializer(parent).data - return None - @extend_schema_field(DocIdentifierSerializer(many=True)) def get_identifiers(self, doc: Document): identifiers = [] From 47d3734955071d1ccc54787698e751c74ce4d303 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Mar 2026 12:32:36 -0400 Subject: [PATCH 025/188] chore(deps): bump types-pytz from 2025.2.0.20250809 to 2025.2.0.20251108 (#10424) Bumps [types-pytz](https://github.com/typeshed-internal/stub_uploader) from 2025.2.0.20250809 to 2025.2.0.20251108. - [Commits](https://github.com/typeshed-internal/stub_uploader/commits) --- updated-dependencies: - dependency-name: types-pytz dependency-version: 2025.2.0.20251108 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cb583d5dc97..3d54b104ee6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -74,7 +74,7 @@ python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=2.0.0 # from TastyPie pytz==2025.2 # Pinned as changes need to be vetted for their effect on Meeting fields -types-pytz==2025.2.0.20250809 # match pytz version +types-pytz==2025.2.0.20251108 # match pytz version requests>=2.32.4 types-requests>=2.32.4 requests-mock>=1.12.1 From 1799245dc6ce82301b0790412957ccfa19910dc1 Mon Sep 17 00:00:00 2001 From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:45:46 +0000 Subject: [PATCH 026/188] ci: update base image target version to 20260304T1633 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 71370fabee5..ce1828052ef 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260211T1901 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260304T1633 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 947f3790e4d..6be54fb6b07 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260211T1901 +20260304T1633 From 7f28542c82e2c51210daf77ca10f9682c0ea709d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 9 Mar 2026 14:02:57 -0300 Subject: [PATCH 027/188] fix: finish dropping email as RfcAuthor field (#10512) * fix: fix admin / Document.author_list() * fix: update RfcAuthorResource email is still accessible, but read only * fix: admin search by RfcAuthor email --- ietf/doc/admin.py | 4 +++- ietf/doc/models.py | 16 +++++++++++----- ietf/doc/resources.py | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index b604d4f0963..0d04e8db3ad 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -241,7 +241,9 @@ def is_deleted(self, instance): admin.site.register(StoredObject, StoredObjectAdmin) class RfcAuthorAdmin(admin.ModelAdmin): + # the email field in the list_display/readonly_fields works through a @property list_display = ['id', 'document', 'titlepage_name', 'person', 'email', 'affiliation', 'country', 'order'] - search_fields = ['document__name', 'titlepage_name', 'person__name', 'email', 'affiliation', 'country'] + search_fields = ['document__name', 'titlepage_name', 'person__name', 'person__email__address', 'affiliation', 'country'] raw_id_fields = ["document", "person"] + readonly_fields = ["email"] admin.site.register(RfcAuthor, RfcAuthorAdmin) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index f1b319367e5..868bc4ac47c 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -466,11 +466,12 @@ def author_persons(self): def author_list(self): """List of author emails""" - author_qs = ( - self.rfcauthor_set - if self.type_id == "rfc" and self.rfcauthor_set.exists() - else self.documentauthor_set - ).select_related("email").order_by("order") + if self.type_id == "rfc" and self.rfcauthor_set.exists(): + author_qs = self.rfcauthor_set.select_related("person").order_by("order") + else: + author_qs = self.documentauthor_set.select_related("email").order_by( + "order" + ) best_addresses = [] for author in author_qs: if author.email: @@ -953,6 +954,11 @@ class Meta: @property def email(self) -> Email | None: return self.person.email() if self.person else None + + def format_for_titlepage(self): + if self.is_editor: + return f"{self.titlepage_name}, Ed." + return self.titlepage_name class DocumentAuthorInfo(models.Model): diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index 556465a522f..1d86df78d0b 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -897,7 +897,7 @@ class Meta: class RfcAuthorResource(ModelResource): document = ToOneField(DocumentResource, 'document') person = ToOneField(PersonResource, 'person', null=True) - email = ToOneField(EmailResource, 'email', null=True) + email = ToOneField(EmailResource, 'email', null=True, readonly=True) class Meta: queryset = RfcAuthor.objects.all() serializer = api.Serializer() From 809e7682db30279cb715f47c89ae546e320c9c76 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 11 Mar 2026 13:07:27 -0500 Subject: [PATCH 028/188] chore: remove task explorer from devcontainer (#10532) --- .devcontainer/devcontainer.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2cfff788538..e4964e8909c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -32,7 +32,6 @@ "mutantdino.resourcemonitor", "oderwat.indent-rainbow", "redhat.vscode-yaml", - "spmeesseman.vscode-taskexplorer", "ms-python.pylint", "charliermarsh.ruff" ], From d4a594ddd4a9dd0bd575465748627f7fea68aac3 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 11 Mar 2026 15:12:22 -0300 Subject: [PATCH 029/188] feat: rfc-index generation (#10526) * chore: stand-in red_bucket STORAGE for dev * feat: rfc index text generation (WIP) Text generation works. Also includes XML generation that is not yet converted. Based on Kesara's implementations in the purple repo. * feat: rfc index XML generation (WIP) * feat: Document.keywords + migration * feat: keywords API * feat: keywords in rfc-index.xml * fix: better stream/area/wg_acronym Still some disagreements, not sure if that's data or logic driven * fix: NON WORKING GROUP logic May need more attention * fix: add rev to draft name * fix: interleave unpublished RFC records * fix: lint * refactor: use lxml * fix: multi-paragraph abstracts * feat: RFCINDEX_MATCH_LEGACY_XML option * fix: zero pad DOIs * fix: better NON WORKING GROUP id * fix: reorder elements * refactor: extract repeated code * refactor: unify DOI generation * fix: modern DOI proxy URL for ATOM feed * refactor: settings.RFC_EDITOR_ERRATA_BASE_URL Drop unused settings.RFC_EDITOR_ERRATA_URL * chore: real red_bucket storage cfg * fix: handle missing json for prod/dev/test * chore: straighten out S3 saving * chore(dev): FileSystemStorage for red_bucket dev (commented out) * chore: configurable bucket path for JSON inputs * test: tests_rfcindex.py Not great coverage, but exercises the generators a bit. * fix: lint + consistent var naming * test: improve test coverage / testability * fix: lint --- docker/configs/settings_local.py | 11 + ietf/api/serializers_rpc.py | 2 + ietf/doc/api.py | 6 - ietf/doc/factories.py | 6 + ietf/doc/feeds.py | 9 +- ...3_dochistory_keywords_document_keywords.py | 31 ++ ietf/doc/models.py | 21 + ietf/doc/serializers.py | 4 +- ietf/doc/views_doc.py | 4 +- ietf/settings.py | 10 +- ietf/settings_test.py | 8 +- ietf/sync/rfcindex.py | 480 ++++++++++++++++++ ietf/sync/tests_rfcindex.py | 230 +++++++++ ietf/templates/sync/rfc-index.txt | 69 +++ k8s/settings_local.py | 33 ++ 15 files changed, 907 insertions(+), 17 deletions(-) create mode 100644 ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py create mode 100644 ietf/sync/rfcindex.py create mode 100644 ietf/sync/tests_rfcindex.py create mode 100644 ietf/templates/sync/rfc-index.txt diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 1d4e6916b94..94adc516a47 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -101,6 +101,17 @@ ), } +# For dev on rfc-index generation, create a red_bucket/ directory in the project root +# and uncomment these settings. Generated files will appear in this directory. To +# generate an accurate index, put up-to-date copies of unusable-rfc-numbers.json, +# april-first-rfc-numbers.json, and publication-std-levels.json in this directory +# before generating the index. +# +# STORAGES["red_bucket"] = { +# "BACKEND": "django.core.files.storage.FileSystemStorage", +# "OPTIONS": {"location": "red_bucket"}, +# } + APP_API_TOKENS = { "ietf.api.red_api" : ["devtoken", "redtoken"], # Not a real secret "ietf.api.views_rpc" : ["devtoken"], # Not a real secret diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index e51b917be4a..c17cbc64cee 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -300,6 +300,7 @@ class Meta: "obsoletes", "updates", "subseries", + "keywords", ] def validate(self, data): @@ -540,6 +541,7 @@ class Meta: "pages", "std_level", "subseries", + "keywords", ] def create(self, validated_data): diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 75993f463eb..73fff6b27ff 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -4,13 +4,11 @@ from django.db.models import ( BooleanField, Count, - JSONField, OuterRef, Prefetch, Q, QuerySet, Subquery, - Value, ) from django.db.models.functions import TruncDate from django_filters import rest_framework as filters @@ -160,10 +158,6 @@ def augment_rfc_queryset(queryset: QuerySet[Document]): output_field=BooleanField(), ) ) - .annotate( - # TODO implement this fake field for real - keywords=Value(["keyword"], output_field=JSONField()), - ) ) diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index bc387654460..1a178c6f31d 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -311,6 +311,12 @@ class Meta: def desc(self): return 'New version available %s-%s'%(self.doc.name,self.rev) +class PublishedRfcDocEventFactory(DocEventFactory): + class Meta: + model = DocEvent + type = "published_rfc" + doc = factory.SubFactory(WgRfcFactory) + class StateDocEventFactory(DocEventFactory): class Meta: model = StateDocEvent diff --git a/ietf/doc/feeds.py b/ietf/doc/feeds.py index 500ed3cb182..afe96cf0dfd 100644 --- a/ietf/doc/feeds.py +++ b/ietf/doc/feeds.py @@ -1,5 +1,4 @@ -# Copyright The IETF Trust 2007-2020, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2007-2026, All Rights Reserved import debug # pyflakes:ignore @@ -263,9 +262,11 @@ def item_extra_kwargs(self, item): ) extra.update({"media_contents": media_contents}) - extra.update({"doi": "10.17487/%s" % item.name.upper()}) extra.update( - {"doiuri": "http://dx.doi.org/10.17487/%s" % item.name.upper()} + { + "doi": item.doi, + "doiuri": f"https://doi.org/{item.doi}", + } ) # R104 Publisher (Mandatory - but we need a string from them first) diff --git a/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py b/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py new file mode 100644 index 00000000000..5e2513e15a9 --- /dev/null +++ b/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models +import ietf.doc.models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0032_remove_rfcauthor_email"), + ] + + operations = [ + migrations.AddField( + model_name="dochistory", + name="keywords", + field=models.JSONField( + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + migrations.AddField( + model_name="document", + name="keywords", + field=models.JSONField( + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 868bc4ac47c..7b23a62c45b 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -13,6 +13,7 @@ from io import BufferedReader from pathlib import Path +from django.core.exceptions import ValidationError from django.db.models import Q from lxml import etree from typing import Optional, Protocol, TYPE_CHECKING, Union @@ -109,6 +110,15 @@ class Meta: IESG_STATCHG_CONFLREV_ACTIVE_STATES = ("iesgeval", "defer") IESG_SUBSTATE_TAGS = ('ad-f-up', 'need-rev', 'extpty') + +def validate_doc_keywords(value): + if ( + not isinstance(value, list | tuple | set) + or not all(isinstance(elt, str) for elt in value) + ): + raise ValidationError("Value must be an array of strings") + + class DocumentInfo(models.Model): """Any kind of document. Draft, RFC, Charter, IPR Statement, Liaison Statement""" time = models.DateTimeField(default=timezone.now) # should probably have auto_now=True @@ -142,6 +152,17 @@ class DocumentInfo(models.Model): uploaded_filename = models.TextField(blank=True) note = models.TextField(blank=True) rfc_number = models.PositiveIntegerField(blank=True, null=True) # only valid for type="rfc" + keywords = models.JSONField( + default=list, + max_length=1000, + validators=[validate_doc_keywords], + ) + + @property + def doi(self) -> str | None: + if self.type_id == "rfc" and self.rfc_number is not None: + return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number:04d}" + return None def file_extension(self): if not hasattr(self, '_cached_extension'): diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index 139ae9aa7e6..3651670962b 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -291,9 +291,9 @@ def get_authors(self, doc: Document): @extend_schema_field(DocIdentifierSerializer(many=True)) def get_identifiers(self, doc: Document): identifiers = [] - if doc.rfc_number: + if doc.doi: identifiers.append( - DocIdentifier(type="doi", value=f"10.17487/RFC{doc.rfc_number:04d}") + DocIdentifier(type="doi", value=doc.doi) ) return DocIdentifierSerializer(instance=identifiers, many=True).data diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 0ae7520681d..c1f6352ac3f 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -1285,9 +1285,7 @@ def document_bibtex(request, name, rev=None): break elif doc.type_id == "rfc": - # This needs to be replaced with a lookup, as the mapping may change - # over time. - doi = f"10.17487/RFC{doc.rfc_number:04d}" + doi = doc.doi if doc.is_dochistory(): latest_event = doc.latest_event(type='new_revision', rev=rev) diff --git a/ietf/settings.py b/ietf/settings.py index 71b110d762a..e0b4f201182 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -838,6 +838,11 @@ def skip_unreadable_post(record): "slides", ] +# Other storages +STORAGES["red_bucket"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "red_bucket"}, +} # Override this in settings_local.py if needed # *_PATH variables ends with a slash/ . @@ -932,10 +937,11 @@ def skip_unreadable_post(record): RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" RFC_EDITOR_ERRATA_JSON_URL = "https://www.rfc-editor.org/errata.json" -RFC_EDITOR_ERRATA_URL = "https://www.rfc-editor.org/errata_search.php?rfc={rfc_number}" RFC_EDITOR_INLINE_ERRATA_URL = "https://www.rfc-editor.org/rfc/inline-errata/rfc{rfc_number}.html" +RFC_EDITOR_ERRATA_BASE_URL = "https://www.rfc-editor.org/errata/" RFC_EDITOR_INFO_BASE_URL = "https://www.rfc-editor.org/info/" + # NomCom Tool settings ROLODEX_URL = "" NOMCOM_PUBLIC_KEYS_DIR = '/a/www/nomcom/public_keys/' @@ -1570,3 +1576,5 @@ def skip_unreadable_post(record): YOUTUBE_DOMAINS = ['www.youtube.com', 'youtube.com', 'youtu.be', 'm.youtube.com', 'youtube-nocookie.com', 'www.youtube-nocookie.com'] + +IETF_DOI_PREFIX = "10.17487" diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 6479069db02..1f5a7e8ddca 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -14,7 +14,7 @@ import shutil import tempfile from ietf.settings import * # pyflakes:ignore -from ietf.settings import ORIG_AUTH_PASSWORD_VALIDATORS +from ietf.settings import ORIG_AUTH_PASSWORD_VALIDATORS, STORAGES import debug # pyflakes:ignore debug.debug = True @@ -114,3 +114,9 @@ def tempdir_with_cleanup(**kwargs): AUTH_PASSWORD_VALIDATORS = ORIG_AUTH_PASSWORD_VALIDATORS except NameError: pass + +# Use InMemoryStorage for red bucket storage +STORAGES["red_bucket"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "red_bucket"}, +} diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py new file mode 100644 index 00000000000..b15846094ff --- /dev/null +++ b/ietf/sync/rfcindex.py @@ -0,0 +1,480 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import json +from collections import defaultdict +from collections.abc import Container +from dataclasses import dataclass +from io import StringIO, BytesIO +from itertools import chain +from operator import attrgetter, itemgetter +from pathlib import Path +from textwrap import fill +from urllib.parse import urljoin + +from django.conf import settings +from lxml import etree + +from django.core.files.storage import storages +from django.db import models +from django.db.models.functions import Substr, Cast +from django.template.loader import render_to_string +from django.utils import timezone + +from ietf.doc.models import Document +from ietf.name.models import StdLevelName +from ietf.utils.log import log + +FORMATS_FOR_INDEX = ["txt", "html", "pdf", "xml", "ps"] + + +def format_rfc_number(n): + """Format an RFC number (or subseries doc number) + + Set settings.RFCINDEX_MATCH_LEGACY_XML=True for the legacy (leading-zero) format. + That is for debugging only - tests will fail. + """ + if getattr(settings, "RFCINDEX_MATCH_LEGACY_XML", False): + return format(n, "04") + else: + return format(n) + + +def errata_url(rfc: Document): + return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}") + + +def save_to_red_bucket(filename: str, content: BytesIO | StringIO): + red_bucket = storages["red_bucket"] + bucket_path = str(Path(getattr(settings, "RFCINDEX_OUTPUT_PATH", "")) / filename) + if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True): + # Django 4.2's FileSystemStorage does not support allow_overwrite. + red_bucket.delete(bucket_path) + red_bucket.save(bucket_path, content) + log(f"Saved {bucket_path} in red_bucket storage") + + +@dataclass +class UnusableRfcNumber: + rfc_number: int + comment: str + + +def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: + FILENAME = "unusable-rfc-numbers.json" + bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + return [] # pragma: no cover + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE == "development": + return [] # pragma: no cover + raise + assert all(isinstance(record["number"], int) for record in records) + assert all(isinstance(record["comment"], str) for record in records) + return [ + UnusableRfcNumber(rfc_number=record["number"], comment=record["comment"]) + for record in sorted(records, key=itemgetter("number")) + ] + + +def get_april1_rfc_numbers() -> Container[int]: + FILENAME = "april-first-rfc-numbers.json" + bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + return [] # pragma: no cover + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE == "development": + return [] # pragma: no cover + raise + assert all(isinstance(record, int) for record in records) + return records + + +def get_publication_std_levels() -> dict[int, StdLevelName]: + FILENAME = "publication-std-levels.json" + bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + values: dict[int, StdLevelName] = {} + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + # intentionally fall through instead of return here + else: + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE != "development": + raise + else: + assert all(isinstance(record["number"], int) for record in records) + values = { + record["number"]: StdLevelName.objects.get( + slug=record["publication_std_level"] + ) + for record in records + } + # defaultdict to return "unknown" for any missing values + unknown_std_level = StdLevelName.objects.get(slug="unkn") + return defaultdict(lambda: unknown_std_level, values) + + +def format_ordering(rfc_number): + if rfc_number < 8650: + ordering = ["txt", "ps", "pdf", "html", "xml"] + else: + ordering = ["html", "txt", "ps", "pdf", "xml"] + return ordering.index # return the method + + +def get_rfc_text_index_entries(): + """Returns RFC entries for rfc-index.txt""" + entries = [] + april1_rfc_numbers = get_april1_rfc_numbers() + published_rfcs = Document.objects.filter(type_id="rfc").order_by("rfc_number") + rfcs = sorted( + chain(published_rfcs, get_unusable_rfc_numbers()), key=attrgetter("rfc_number") + ) + for rfc in rfcs: + if isinstance(rfc, UnusableRfcNumber): + entries.append(f"{format_rfc_number(rfc.rfc_number)} Not Issued.") + else: + assert isinstance(rfc, Document) + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + published_at = rfc.pub_date() + date = ( + published_at.strftime("1 %B %Y") + if rfc.rfc_number in april1_rfc_numbers + else published_at.strftime("%B %Y") + ) + + # formats + formats = ", ".join( + sorted( + [ + format["fmt"] + for format in rfc.formats() + if format["fmt"] in FORMATS_FOR_INDEX + ], + key=format_ordering(rfc.rfc_number), + ) + ).upper() + + # obsoletes + obsoletes = "" + obsoletes_documents = sorted( + rfc.related_that_doc("obs"), + key=attrgetter("rfc_number"), + ) + if len(obsoletes_documents) > 0: + obsoletes_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in obsoletes_documents + ) + obsoletes = f" (Obsoletes {obsoletes_names})" + + # obsoleted by + obsoleted_by = "" + obsoleted_by_documents = sorted( + rfc.related_that("obs"), + key=attrgetter("rfc_number"), + ) + if len(obsoleted_by_documents) > 0: + obsoleted_by_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in obsoleted_by_documents + ) + obsoleted_by = f" (Obsoleted by {obsoleted_by_names})" + + # updates + updates = "" + updates_documents = sorted( + rfc.related_that_doc("updates"), + key=attrgetter("rfc_number"), + ) + if len(updates_documents) > 0: + updates_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in updates_documents + ) + updates = f" (Updates {updates_names})" + + # updated by + updated_by = "" + updated_by_documents = sorted( + rfc.related_that("updates"), + key=attrgetter("rfc_number"), + ) + if len(updated_by_documents) > 0: + updated_by_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in updated_by_documents + ) + updated_by = f" (Updated by {updated_by_names})" + + doc_relations = f"{obsoletes}{obsoleted_by}{updates}{updated_by} " + + # subseries + subseries = ",".join( + f"{container.type.slug}{format_rfc_number(int(container.name[3:]))}" + for container in rfc.part_of() + ).upper() + if subseries: + subseries = f"(Also {subseries}) " + + entry = fill( + ( + f"{format_rfc_number(rfc.rfc_number)} {rfc.title}. {authors}. {date}. " + f"(Format: {formats}){doc_relations}{subseries}" + f"(Status: {str(rfc.std_level).upper()}) " + f"(DOI: {rfc.doi})" + ), + width=73, + subsequent_indent=" " * 5, + ) + entries.append(entry) + + return entries + + +def add_subseries_xml_index_entries(rfc_index, ss_type, include_all=False): + """Add subseries entries for rfc-index.xml""" + # subseries docs annotated with numeric number + ss_docs = list( + Document.objects.filter(type_id=ss_type) + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + ) + if len(ss_docs) == 0: + return # very much not expected + highest_number = ss_docs[0].number + for ss_number in range(1, highest_number + 1): + if ss_docs[-1].number == ss_number: + this_ss_doc = ss_docs.pop() + contained_rfcs = this_ss_doc.contains() + else: + contained_rfcs = [] + if len(contained_rfcs) == 0 and not include_all: + continue + entry = etree.SubElement(rfc_index, f"{ss_type}-entry") + etree.SubElement( + entry, "doc-id" + ).text = f"{ss_type.upper()}{format_rfc_number(ss_number)}" + if len(contained_rfcs) > 0: + is_also = etree.SubElement(entry, "is-also") + for rfc in sorted(contained_rfcs, key=attrgetter("rfc_number")): + etree.SubElement( + is_also, "doc-id" + ).text = f"RFC{format_rfc_number(rfc.rfc_number)}" + + +def add_related_xml_index_entries(root: etree.Element, rfc: Document, tag: str): + relation_getter = { + "obsoletes": lambda doc: doc.related_that_doc("obs"), + "obsoleted-by": lambda doc: doc.related_that("obs"), + "updates": lambda doc: doc.related_that_doc("updates"), + "updated-by": lambda doc: doc.related_that("updates"), + } + related_docs = sorted( + relation_getter[tag](rfc), + key=attrgetter("rfc_number"), + ) + if len(related_docs) > 0: + element = etree.SubElement(root, tag) + for doc in related_docs: + etree.SubElement( + element, "doc-id" + ).text = f"RFC{format_rfc_number(doc.rfc_number)}" + + +def add_rfc_xml_index_entries(rfc_index): + """Add RFC entries for rfc-index.xml""" + entries = [] + april1_rfc_numbers = get_april1_rfc_numbers() + publication_statuses = get_publication_std_levels() + + published_rfcs = Document.objects.filter(type_id="rfc").order_by("rfc_number") + + # Iterators for unpublished and published, both sorted by number + unpublished_iter = iter(get_unusable_rfc_numbers()) + published_iter = iter(published_rfcs) + + # Prime the next_* values + next_unpublished = next(unpublished_iter, None) + next_published = next(published_iter, None) + + while next_published is not None or next_unpublished is not None: + if next_unpublished is not None and ( + next_published is None + or next_unpublished.rfc_number < next_published.rfc_number + ): + entry = etree.SubElement(rfc_index, "rfc-not-issued-entry") + etree.SubElement( + entry, "doc-id" + ).text = f"RFC{format_rfc_number(next_unpublished.rfc_number)}" + entries.append(entry) + next_unpublished = next(unpublished_iter, None) + continue + + rfc = next_published # hang on to this + next_published = next(published_iter, None) # prep for next iteration + entry = etree.SubElement(rfc_index, "rfc-entry") + + etree.SubElement( + entry, "doc-id" + ).text = f"RFC{format_rfc_number(rfc.rfc_number)}" + etree.SubElement(entry, "title").text = rfc.title + + for author in rfc.rfcauthor_set.all(): + author_element = etree.SubElement(entry, "author") + etree.SubElement(author_element, "name").text = author.titlepage_name + if author.is_editor: + etree.SubElement(author_element, "title").text = "Editor" + + date = etree.SubElement(entry, "date") + published_at = rfc.pub_date() + etree.SubElement(date, "month").text = published_at.strftime("%B") + if rfc.rfc_number in april1_rfc_numbers: + etree.SubElement(date, "day").text = str(published_at.day) + etree.SubElement(date, "year").text = str(published_at.year) + + format_ = etree.SubElement(entry, "format") + fmts = [ff["fmt"] for ff in rfc.formats() if ff["fmt"] in FORMATS_FOR_INDEX] + for fmt in sorted(fmts, key=format_ordering(rfc.rfc_number)): + match_legacy = getattr(settings, "RFCINDEX_MATCH_LEGACY_XML", False) + etree.SubElement(format_, "file-format").text = ( + "ASCII" if match_legacy and fmt == "txt" else fmt.upper() + ) + + etree.SubElement(entry, "page-count").text = str(rfc.pages) + + if len(rfc.keywords) > 0: + keywords = etree.SubElement(entry, "keywords") + for keyword in rfc.keywords: + etree.SubElement(keywords, "kw").text = keyword.strip() + + if rfc.abstract: + abstract = etree.SubElement(entry, "abstract") + for paragraph in rfc.abstract.split("\n\n"): + etree.SubElement(abstract, "p").text = paragraph.strip() + + draft = rfc.came_from_draft() + if draft is not None: + etree.SubElement(entry, "draft").text = f"{draft.name}-{draft.rev}" + + part_of_documents = rfc.part_of() + if len(part_of_documents) > 0: + is_also = etree.SubElement(entry, "is-also") + for doc in part_of_documents: + etree.SubElement(is_also, "doc-id").text = doc.name.upper() + + add_related_xml_index_entries(entry, rfc, "obsoletes") + add_related_xml_index_entries(entry, rfc, "obsoleted-by") + add_related_xml_index_entries(entry, rfc, "updates") + add_related_xml_index_entries(entry, rfc, "updated-by") + + etree.SubElement(entry, "current-status").text = rfc.std_level.name.upper() + etree.SubElement(entry, "publication-status").text = publication_statuses[ + rfc.rfc_number + ].name.upper() + etree.SubElement(entry, "stream").text = ( + "INDEPENDENT" if rfc.stream_id == "ise" else rfc.stream.name + ) + + # Add area / wg_acronym + if rfc.stream_id == "ietf": + if rfc.group.type_id in ["individ", "area"]: + etree.SubElement(entry, "wg_acronym").text = "NON WORKING GROUP" + else: + if rfc.area is not None: + etree.SubElement(entry, "area").text = rfc.area.acronym + if rfc.group: + etree.SubElement(entry, "wg_acronym").text = rfc.group.acronym + + if rfc.tags.filter(slug="errata").exists(): + etree.SubElement(entry, "errata-url").text = errata_url(rfc) + etree.SubElement(entry, "doi").text = rfc.doi + entries.append(entry) + + +def create_rfc_txt_index(): + """Create text index of published documents""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating rfc-index.txt") + index = render_to_string( + "sync/rfc-index.txt", + { + "created_on": created_on, + "rfcs": get_rfc_text_index_entries(), + }, + ) + save_to_red_bucket("rfc-index.txt", StringIO(index)) + + +def create_rfc_xml_index(): + """Create XML index of published documents""" + XSI_NAMESPACE = "http://www.w3.org/2001/XMLSchema-instance" + XSI = "{" + XSI_NAMESPACE + "}" + + log("Creating rfc-index.xml") + rfc_index = etree.Element( + "rfc-index", + nsmap={ + None: "https://www.rfc-editor.org/rfc-index", + "xsi": XSI_NAMESPACE, + }, + attrib={ + XSI + "schemaLocation": ( + "https://www.rfc-editor.org/rfc-index " + "https://www.rfc-editor.org/rfc-index.xsd" + ), + }, + ) + + # add data + add_subseries_xml_index_entries(rfc_index, "bcp", include_all=True) + add_subseries_xml_index_entries(rfc_index, "fyi") + add_rfc_xml_index_entries(rfc_index) + add_subseries_xml_index_entries(rfc_index, "std") + + # make it pretty + pretty_index = etree.tostring( + rfc_index, + encoding="utf-8", + xml_declaration=True, + pretty_print=4, + ) + save_to_red_bucket("rfc-index.xml", BytesIO(pretty_index)) diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py new file mode 100644 index 00000000000..b0a8712fe17 --- /dev/null +++ b/ietf/sync/tests_rfcindex.py @@ -0,0 +1,230 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import json +from io import BytesIO, StringIO +from unittest import mock + +from django.core.files.storage import storages +from django.test.utils import override_settings +from lxml import etree + +from ietf.doc.factories import PublishedRfcDocEventFactory, IndividualRfcFactory +from ietf.name.models import DocTagName +from ietf.sync.rfcindex import ( + create_rfc_txt_index, + create_rfc_xml_index, + format_rfc_number, + save_to_red_bucket, get_unusable_rfc_numbers, get_april1_rfc_numbers, + get_publication_std_levels, +) +from ietf.utils.test_utils import TestCase + + +class RfcIndexTests(TestCase): + """Tests of rfc-index generation + + Tests are limited and should cover more cases. Needs: + * test of subseries docs + * test of related docs (obsoletes/updates + reverse directions) + * more thorough validation of index contents + + Be careful when calling create_rfc_txt_index() or create_rfc_xml_index(). These + will save to a storage by default, which can introduce cross-talk between tests. + Best to patch that method with a mock. + """ + + def setUp(self): + super().setUp() + red_bucket = storages["red_bucket"] + + # Create an unused RFC number + red_bucket.save( + "input/unusable-rfc-numbers.json", + StringIO(json.dumps([{"number": 123, "comment": ""}])), + ) + + # actual April 1 RFC + self.april_fools_rfc = PublishedRfcDocEventFactory( + time="2020-04-01T12:00:00Z", + doc=IndividualRfcFactory( + name="rfc4560", + rfc_number=4560, + stream_id="ise", + std_level_id="inf", + ), + ).doc + # Set up a JSON file to flag the April 1 RFC + red_bucket.save( + "input/april-first-rfc-numbers.json", + StringIO(json.dumps([self.april_fools_rfc.rfc_number])), + ) + + # non-April Fools RFC that happens to have been published on April 1 + self.rfc = PublishedRfcDocEventFactory( + time="2021-04-01T12:00:00Z", + doc__name="rfc10000", + doc__rfc_number=10000, + doc__std_level_id="std", + ).doc + self.rfc.tags.add(DocTagName.objects.get(slug="errata")) + + # Set up a publication-std-levels.json file to indicate the publication + # standard of self.rfc as different from its current value + red_bucket.save( + "input/publication-std-levels.json", + StringIO( + json.dumps( + [{"number": self.rfc.rfc_number, "publication_std_level": "ps"}] + ) + ), + ) + + def tearDown(self): + red_bucket = storages["red_bucket"] + red_bucket.delete("input/unusable-rfc-numbers.json") + red_bucket.delete("input/april-first-rfc-numbers.json") + red_bucket.delete("input/publication-std-levels.json") + super().tearDown() + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_rfc_txt_index(self, mock_save): + create_rfc_txt_index() + self.assertEqual(mock_save.call_count, 1) + self.assertEqual(mock_save.call_args[0][0], "rfc-index.txt") + contents = mock_save.call_args[0][1].read() + self.assertIn( + "123 Not Issued.", + contents, + ) + # No zero prefix! + self.assertNotIn( + "0123 Not Issued.", + contents, + ) + self.assertIn( + f"{self.april_fools_rfc.rfc_number} {self.april_fools_rfc.title}", + contents, + ) + self.assertIn("1 April 2020", contents) # from the April 1 RFC + self.assertIn( + f"{self.rfc.rfc_number} {self.rfc.title}", + contents, + ) + self.assertIn("April 2021", contents) # from the non-April 1 RFC + self.assertNotIn("1 April 2021", contents) + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_rfc_xml_index(self, mock_save): + create_rfc_xml_index() + self.assertEqual(mock_save.call_count, 1) + self.assertEqual(mock_save.call_args[0][0], "rfc-index.xml") + contents = mock_save.call_args[0][1].read() + ns = "{https://www.rfc-editor.org/rfc-index}" # NOT an f-string + index = etree.fromstring(contents) + + # We can aspire to validating the schema - currently does not conform because + # XSD expects 4-digit RFC numbers (etc). + # + # xmlschema = etree.XMLSchema(etree.fromstring( + # Path(__file__).with_name("rfc-index.xsd").read_bytes()) + # ) + # xmlschema.assertValid(index) + + children = list(index) # elements as list + # Should be one rfc-not-issued-entry + self.assertEqual(len(children), 3) + self.assertEqual( + [ + c.find(f"{ns}doc-id").text + for c in children + if c.tag == f"{ns}rfc-not-issued-entry" + ], + ["RFC123"], + ) + # Should be two rfc-entries + rfc_entries = { + c.find(f"{ns}doc-id").text: c for c in children if c.tag == f"{ns}rfc-entry" + } + + # Check the April Fool's entry + april_fools_entry = rfc_entries[self.april_fools_rfc.name.upper()] + self.assertEqual( + april_fools_entry.find(f"{ns}title").text, + self.april_fools_rfc.title, + ) + self.assertEqual( + [(c.tag, c.text) for c in april_fools_entry.find(f"{ns}date")], + [(f"{ns}month", "April"), (f"{ns}day", "1"), (f"{ns}year", "2020")], + ) + self.assertEqual( + april_fools_entry.find(f"{ns}current-status").text, + "INFORMATIONAL", + ) + self.assertEqual( + april_fools_entry.find(f"{ns}publication-status").text, + "UNKNOWN", + ) + + # Check the Regular entry + rfc_entry = rfc_entries[self.rfc.name.upper()] + self.assertEqual(rfc_entry.find(f"{ns}title").text, self.rfc.title) + self.assertEqual( + rfc_entry.find(f"{ns}current-status").text, "INTERNET STANDARD" + ) + self.assertEqual( + rfc_entry.find(f"{ns}publication-status").text, "PROPOSED STANDARD" + ) + self.assertEqual( + [(c.tag, c.text) for c in rfc_entry.find(f"{ns}date")], + [(f"{ns}month", "April"), (f"{ns}year", "2021")], + ) + + +class HelperTests(TestCase): + def test_format_rfc_number(self): + self.assertEqual(format_rfc_number(10), "10") + with override_settings(RFCINDEX_MATCH_LEGACY_XML=True): + self.assertEqual(format_rfc_number(10), "0010") + + def test_save_to_red_bucket(self): + red_bucket = storages["red_bucket"] + with override_settings(RFCINDEX_DELETE_THEN_WRITE=False): + save_to_red_bucket("test", StringIO("contents")) + with red_bucket.open("test", "r") as f: + self.assertEqual(f.read(), "contents") + with override_settings(RFCINDEX_DELETE_THEN_WRITE=True): + save_to_red_bucket("test", BytesIO(b"new contents")) + with red_bucket.open("test", "r") as f: + self.assertEqual(f.read(), "new contents") + red_bucket.delete("test") # clean up like a good child + + def test_get_unusable_rfc_numbers_raises(self): + """get_unusable_rfc_numbers should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_unusable_rfc_numbers() + red_bucket = storages["red_bucket"] + red_bucket.save("unusable-rfc-numbers.json", StringIO("not json")) + with self.assertRaises(json.JSONDecodeError): + get_unusable_rfc_numbers() + red_bucket.delete("unusable-rfc-numbers.json") + + def test_get_april1_rfc_numbers_raises(self): + """get_april1_rfc_numbers should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_april1_rfc_numbers() + red_bucket = storages["red_bucket"] + red_bucket.save("april-first-rfc-numbers.json", StringIO("not json")) + with self.assertRaises(json.JSONDecodeError): + get_april1_rfc_numbers() + red_bucket.delete("april-first-rfc-numbers.json") + + def test_get_publication_std_levels_raises(self): + """get_publication_std_levels should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_publication_std_levels() + red_bucket = storages["red_bucket"] + red_bucket.save("publication-std-levels.json", StringIO("not json")) + with self.assertRaises(json.JSONDecodeError): + get_publication_std_levels() + red_bucket.delete("publication-std-levels.json") diff --git a/ietf/templates/sync/rfc-index.txt b/ietf/templates/sync/rfc-index.txt new file mode 100644 index 00000000000..0f01ddfa905 --- /dev/null +++ b/ietf/templates/sync/rfc-index.txt @@ -0,0 +1,69 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + RFC INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all RFCs in numeric order. + +RFC citations appear in this format: + + #### Title of RFC. Author 1, Author 2, Author 3. Issue date. + (Format: ASCII) (Obsoletes xxx) (Obsoleted by xxx) (Updates xxx) + (Updated by xxx) (Also FYI ####) (Status: ssssss) (DOI: ddd) + +or + + #### Not Issued. + +For example: + + 1129 Internet Time Synchronization: The Network Time Protocol. D.L. + Mills. October 1989. (Format: TXT, PS, PDF, HTML) (Also RFC1119) + (Status: INFORMATIONAL) (DOI: 10.17487/RFC1129) + +Key to citations: + +#### is the RFC number. + +Following the RFC number are the title, the author(s), and the +publication date of the RFC. Each of these is terminated by a period. + +Following the number are the title (terminated with a period), the +author, or list of authors (terminated with a period), and the date +(terminated with a period). + +The format follows in parentheses. One or more of the following formats +are listed: text (TXT), PostScript (PS), Portable Document Format +(PDF), HTML, XML. + +Obsoletes xxxx refers to other RFCs that this one replaces; +Obsoleted by xxxx refers to RFCs that have replaced this one. +Updates xxxx refers to other RFCs that this one merely updates (but +does not replace); Updated by xxxx refers to RFCs that have updated +(but not replaced) this one. Generally, only immediately succeeding +and/or preceding RFCs are indicated, not the entire history of each +related earlier or later RFC in a related series. + +The (Also FYI ##) or (Also STD ##) or (Also BCP ##) phrase gives the +equivalent FYI, STD, or BCP number if the RFC is also in those +document sub-series. The Status field gives the document's +current status (see RFC 2026). The (DOI ddd) field gives the +Digital Object Identifier. + +RFCs may be obtained in a number of ways, using HTTP, FTP, or email. +See the RFC Editor Web page http://www.rfc-editor.org + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + RFC INDEX + --------- + + + +{% for rfc in rfcs %}{{rfc|safe}} + +{% endfor %} diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 0386dbbdf9c..5ca4ba5cd93 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -417,6 +417,39 @@ def _multiline_to_list(s): ), } +# Configure storage for the red bucket - assume it uses the same credentials as +# other blobs +_red_bucket_name = os.environ.get("DATATRACKER_BLOB_STORE_RED_BUCKET_NAME", "").strip() +if _red_bucket_name == "": + raise RuntimeError("DATATRACKER_BLOB_STORE_RED_BUCKET_NAME must be set") + +STORAGES["red_bucket"] = { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": dict( + endpoint_url=_blob_store_endpoint_url, + access_key=_blob_store_access_key, + secret_key=_blob_store_secret_key, + security_token=None, + client_config=botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + signature_version="s3v4", + connect_timeout=_blob_store_connect_timeout, + read_timeout=_blob_store_read_timeout, + retries={"total_max_attempts": _blob_store_max_attempts}, + ), + verify=False, + bucket_name=_red_bucket_name, + ), +} +RFCINDEX_DELETE_THEN_WRITE = False # S3Storage allows file_overwrite by default +RFCINDEX_OUTPUT_PATH = os.environ.get( + "DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/" +) +RFCINDEX_INPUT_PATH = os.environ.get( + "DATATRACKR_RFCINDEX_INPUT_PATH", "" +) + # Configure the blobdb app for artifact storage _blobdb_replication_enabled = ( os.environ.get("DATATRACKER_BLOBDB_REPLICATION_ENABLED", "true").lower() == "true" From c226749c301fbecfd5503ebd66ba3692187d2946 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 12 Mar 2026 16:31:56 -0300 Subject: [PATCH 030/188] feat: task + API for rfc-index creation (#10537) * chore: fix typo in k8s/settings_local.py * feat: refresh_rfc_index() API * fix: use ContentFile, manually encode str Works better with S3Storage * chore(dev): expose blobstore on fixed ports Simplifies connecting purple to the blob store * chore(dev): typo * test: fix + test encoding more carefully * test: cover the new url --- .devcontainer/docker-compose.extend.yml | 4 +-- docker/docker-compose.extend.yml | 4 +-- ietf/api/tests_views_rpc.py | 15 ++++++++++ ietf/api/urls_rpc.py | 5 ++++ ietf/api/views_rpc.py | 16 +++++++++++ ietf/sync/rfcindex.py | 17 ++++++----- ietf/sync/tasks.py | 10 ++++++- ietf/sync/tests_rfcindex.py | 38 ++++++++++++++----------- k8s/settings_local.py | 2 +- 9 files changed, 82 insertions(+), 29 deletions(-) diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml index a92f42bc6d4..ce1ce259fdb 100644 --- a/.devcontainer/docker-compose.extend.yml +++ b/.devcontainer/docker-compose.extend.yml @@ -14,8 +14,8 @@ services: network_mode: service:db blobstore: ports: - - '9000' - - '9001' + - '9000:9000' + - '9001:9001' volumes: datatracker-vscode-ext: diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml index a69a453110c..12ebe447d5c 100644 --- a/docker/docker-compose.extend.yml +++ b/docker/docker-compose.extend.yml @@ -18,8 +18,8 @@ services: - '5433' blobstore: ports: - - '9000' - - '9001' + - '9000:9000' + - '9001:9001' celery: volumes: - .:/workspace diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 6a5a5c9b88e..7ab8778d287 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -363,3 +363,18 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 200) # conflict + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.create_rfc_index_task") + def test_refresh_rfc_index(self, mock_task): + url = urlreverse("ietf.api.purple_api.refresh_rfc_index") + response = self.client.get(url) + self.assertEqual(response.status_code, 403) + response = self.client.get(url, headers={"X-Api-Key": "invalid-token"}) + self.assertEqual(response.status_code, 403) + response = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(response.status_code, 405) + self.assertFalse(mock_task.delay.called) + response = self.client.post(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(response.status_code, 202) + self.assertTrue(mock_task.delay.called) diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index 9d41ac137f2..8555610dc3d 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -30,6 +30,11 @@ views_rpc.RfcPubFilesView.as_view(), name="ietf.api.purple_api.upload_rfc_files", ), + path( + r"rfc_index/refresh/", + views_rpc.RfcIndexView.as_view(), + name="ietf.api.purple_api.refresh_rfc_index", + ), path(r"subject//person/", views_rpc.SubjectPersonView.as_view()), ] diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 8862bbf8660..c7ae6990056 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -40,6 +40,7 @@ from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage from ietf.doc.tasks import signal_update_rfc_metadata_task from ietf.person.models import Email, Person +from ietf.sync.tasks import create_rfc_index_task class Conflict(APIException): @@ -516,3 +517,18 @@ def post(self, request): shutil.move(ftm, destination) return Response(NotificationAckSerializer().data) + + +class RfcIndexView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="refresh_rfc_index", + summary="Refresh rfc-index files", + description="Requests creation of rfc-index.xml and rfc-index.txt files", + responses={202: None}, + request=None, + ) + def post(self, request): + create_rfc_index_task.delay() + return Response(status=202) diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index b15846094ff..63c20449312 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -3,7 +3,6 @@ from collections import defaultdict from collections.abc import Container from dataclasses import dataclass -from io import StringIO, BytesIO from itertools import chain from operator import attrgetter, itemgetter from pathlib import Path @@ -11,6 +10,7 @@ from urllib.parse import urljoin from django.conf import settings +from django.core.files.base import ContentFile from lxml import etree from django.core.files.storage import storages @@ -28,7 +28,7 @@ def format_rfc_number(n): """Format an RFC number (or subseries doc number) - + Set settings.RFCINDEX_MATCH_LEGACY_XML=True for the legacy (leading-zero) format. That is for debugging only - tests will fail. """ @@ -42,13 +42,16 @@ def errata_url(rfc: Document): return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}") -def save_to_red_bucket(filename: str, content: BytesIO | StringIO): +def save_to_red_bucket(filename: str, content: str | bytes): red_bucket = storages["red_bucket"] bucket_path = str(Path(getattr(settings, "RFCINDEX_OUTPUT_PATH", "")) / filename) if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True): # Django 4.2's FileSystemStorage does not support allow_overwrite. red_bucket.delete(bucket_path) - red_bucket.save(bucket_path, content) + red_bucket.save( + bucket_path, + ContentFile(content if isinstance(content, bytes) else content.encode("utf-8")), + ) log(f"Saved {bucket_path} in red_bucket storage") @@ -76,7 +79,7 @@ def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: except json.JSONDecodeError: log(f"Error: unable to parse {bucket_path} in red_bucket storage") if settings.SERVER_MODE == "development": - return [] # pragma: no cover + return [] # pragma: no cover raise assert all(isinstance(record["number"], int) for record in records) assert all(isinstance(record["comment"], str) for record in records) @@ -441,7 +444,7 @@ def create_rfc_txt_index(): "rfcs": get_rfc_text_index_entries(), }, ) - save_to_red_bucket("rfc-index.txt", StringIO(index)) + save_to_red_bucket("rfc-index.txt", index) def create_rfc_xml_index(): @@ -477,4 +480,4 @@ def create_rfc_xml_index(): xml_declaration=True, pretty_print=4, ) - save_to_red_bucket("rfc-index.xml", BytesIO(pretty_index)) + save_to_red_bucket("rfc-index.xml", pretty_index) diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index fc75a056ed7..4c84dc581e5 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # @@ -18,6 +18,7 @@ from ietf.sync import iana from ietf.sync import rfceditor from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue +from ietf.sync.rfcindex import create_rfc_txt_index, create_rfc_xml_index from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper from ietf.utils import log from ietf.utils.timezone import date_today @@ -272,3 +273,10 @@ def load_rfcs_into_blobdb_task(start: int, end: int): if end > 11000: # Arbitrarily chosen end = 11000 load_rfcs_into_blobdb(list(range(start, end + 1))) + + +@shared_task +def create_rfc_index_task(): + create_rfc_txt_index() + create_rfc_xml_index() + diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index b0a8712fe17..e682c016f52 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -1,8 +1,8 @@ # Copyright The IETF Trust 2026, All Rights Reserved import json -from io import BytesIO, StringIO from unittest import mock +from django.core.files.base import ContentFile from django.core.files.storage import storages from django.test.utils import override_settings from lxml import etree @@ -13,7 +13,9 @@ create_rfc_txt_index, create_rfc_xml_index, format_rfc_number, - save_to_red_bucket, get_unusable_rfc_numbers, get_april1_rfc_numbers, + save_to_red_bucket, + get_unusable_rfc_numbers, + get_april1_rfc_numbers, get_publication_std_levels, ) from ietf.utils.test_utils import TestCase @@ -39,7 +41,7 @@ def setUp(self): # Create an unused RFC number red_bucket.save( "input/unusable-rfc-numbers.json", - StringIO(json.dumps([{"number": 123, "comment": ""}])), + ContentFile(json.dumps([{"number": 123, "comment": ""}])), ) # actual April 1 RFC @@ -55,7 +57,7 @@ def setUp(self): # Set up a JSON file to flag the April 1 RFC red_bucket.save( "input/april-first-rfc-numbers.json", - StringIO(json.dumps([self.april_fools_rfc.rfc_number])), + ContentFile(json.dumps([self.april_fools_rfc.rfc_number])), ) # non-April Fools RFC that happens to have been published on April 1 @@ -71,7 +73,7 @@ def setUp(self): # standard of self.rfc as different from its current value red_bucket.save( "input/publication-std-levels.json", - StringIO( + ContentFile( json.dumps( [{"number": self.rfc.rfc_number, "publication_std_level": "ps"}] ) @@ -91,7 +93,8 @@ def test_create_rfc_txt_index(self, mock_save): create_rfc_txt_index() self.assertEqual(mock_save.call_count, 1) self.assertEqual(mock_save.call_args[0][0], "rfc-index.txt") - contents = mock_save.call_args[0][1].read() + contents = mock_save.call_args[0][1] + self.assertTrue(isinstance(contents, str)) self.assertIn( "123 Not Issued.", contents, @@ -119,7 +122,8 @@ def test_create_rfc_xml_index(self, mock_save): create_rfc_xml_index() self.assertEqual(mock_save.call_count, 1) self.assertEqual(mock_save.call_args[0][0], "rfc-index.xml") - contents = mock_save.call_args[0][1].read() + contents = mock_save.call_args[0][1] + self.assertTrue(isinstance(contents, bytes)) ns = "{https://www.rfc-editor.org/rfc-index}" # NOT an f-string index = etree.fromstring(contents) @@ -190,13 +194,15 @@ def test_format_rfc_number(self): def test_save_to_red_bucket(self): red_bucket = storages["red_bucket"] with override_settings(RFCINDEX_DELETE_THEN_WRITE=False): - save_to_red_bucket("test", StringIO("contents")) - with red_bucket.open("test", "r") as f: - self.assertEqual(f.read(), "contents") + save_to_red_bucket("test", "contents \U0001f600") + # Read as binary and explicitly decode to confirm encoding + with red_bucket.open("test", "rb") as f: + self.assertEqual(f.read().decode("utf-8"), "contents \U0001f600") with override_settings(RFCINDEX_DELETE_THEN_WRITE=True): - save_to_red_bucket("test", BytesIO(b"new contents")) - with red_bucket.open("test", "r") as f: - self.assertEqual(f.read(), "new contents") + save_to_red_bucket("test", "new contents \U0001fae0".encode("utf-8")) + # Read as binary and explicitly decode to confirm encoding + with red_bucket.open("test", "rb") as f: + self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0") red_bucket.delete("test") # clean up like a good child def test_get_unusable_rfc_numbers_raises(self): @@ -204,7 +210,7 @@ def test_get_unusable_rfc_numbers_raises(self): with self.assertRaises(FileNotFoundError): get_unusable_rfc_numbers() red_bucket = storages["red_bucket"] - red_bucket.save("unusable-rfc-numbers.json", StringIO("not json")) + red_bucket.save("unusable-rfc-numbers.json", ContentFile("not json")) with self.assertRaises(json.JSONDecodeError): get_unusable_rfc_numbers() red_bucket.delete("unusable-rfc-numbers.json") @@ -214,7 +220,7 @@ def test_get_april1_rfc_numbers_raises(self): with self.assertRaises(FileNotFoundError): get_april1_rfc_numbers() red_bucket = storages["red_bucket"] - red_bucket.save("april-first-rfc-numbers.json", StringIO("not json")) + red_bucket.save("april-first-rfc-numbers.json", ContentFile("not json")) with self.assertRaises(json.JSONDecodeError): get_april1_rfc_numbers() red_bucket.delete("april-first-rfc-numbers.json") @@ -224,7 +230,7 @@ def test_get_publication_std_levels_raises(self): with self.assertRaises(FileNotFoundError): get_publication_std_levels() red_bucket = storages["red_bucket"] - red_bucket.save("publication-std-levels.json", StringIO("not json")) + red_bucket.save("publication-std-levels.json", ContentFile("not json")) with self.assertRaises(json.JSONDecodeError): get_publication_std_levels() red_bucket.delete("publication-std-levels.json") diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 5ca4ba5cd93..56e395c5ace 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -447,7 +447,7 @@ def _multiline_to_list(s): "DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/" ) RFCINDEX_INPUT_PATH = os.environ.get( - "DATATRACKR_RFCINDEX_INPUT_PATH", "" + "DATATRACKER_RFCINDEX_INPUT_PATH", "" ) # Configure the blobdb app for artifact storage From 2c59afe783216285b9695e99ee64547fe4e66469 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 13 Mar 2026 16:14:38 -0300 Subject: [PATCH 031/188] fix: drop stale obs/updates in rfced sync (#10543) * fix: drop stale obs/updates in rfced sync * refactor: partial revert, orig was safer --- ietf/sync/rfceditor.py | 73 +++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py index cdcdeb5989c..aa0e643b20a 100644 --- a/ietf/sync/rfceditor.py +++ b/ietf/sync/rfceditor.py @@ -636,43 +636,70 @@ def update_docs_from_rfc_index( ) rfc_published = True - def parse_relation_list(l): - res = [] - for x in l: - for a in Document.objects.filter(name=x.lower(), type_id="rfc"): - if a not in res: - res.append(a) - return res - - for x in parse_relation_list(obsoletes): - if not RelatedDocument.objects.filter( - source=doc, target=x, relationship=relationship_obsoletes + def parse_relation_list(rel_list: list[str]) -> list[Document]: + return list( + Document.objects.filter( + name__in=[name.strip().lower() for name in rel_list], + type_id="rfc" + ) + ) + + # Create missing obsoletes relations + docs_this_obsoletes = parse_relation_list(obsoletes) + for obs_doc in docs_this_obsoletes: + if not doc.relateddocument_set.filter( + target=obs_doc, relationship=relationship_obsoletes ): - r = RelatedDocument.objects.create( - source=doc, target=x, relationship=relationship_obsoletes + r = doc.relateddocument_set.create( + target=obs_doc, relationship=relationship_obsoletes ) rfc_changes.append( - "created {rel_name} relation between {src_name} and {tgt_name}".format( + "created {rel_name} relation between {src} and {tgt}".format( rel_name=r.relationship.name.lower(), - src_name=prettify_std_name(r.source.name), - tgt_name=prettify_std_name(r.target.name), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), ) ) + # Remove stale obsoletes relations + for r in doc.relateddocument_set.filter( + relationship=relationship_obsoletes + ).exclude(target_id__in=[d.pk for d in docs_this_obsoletes]): + r.delete() + rfc_changes.append( + "removed {rel_name} relation between {src} and {tgt}".format( + rel_name=r.relationship.name.lower(), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), + ) + ) - for x in parse_relation_list(updates): + docs_this_updates = parse_relation_list(updates) + for upd_doc in docs_this_updates: if not RelatedDocument.objects.filter( - source=doc, target=x, relationship=relationship_updates + source=doc, target=upd_doc, relationship=relationship_updates ): - r = RelatedDocument.objects.create( - source=doc, target=x, relationship=relationship_updates + r = doc.relateddocument_set.create( + target=upd_doc, relationship=relationship_updates ) rfc_changes.append( - "created {rel_name} relation between {src_name} and {tgt_name}".format( + "created {rel_name} relation between {src} and {tgt}".format( rel_name=r.relationship.name.lower(), - src_name=prettify_std_name(r.source.name), - tgt_name=prettify_std_name(r.target.name), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), ) ) + # Remove stale updates relations + for r in doc.relateddocument_set.filter( + relationship=relationship_updates + ).exclude(target_id__in=[d.pk for d in docs_this_updates]): + r.delete() + rfc_changes.append( + "removed {rel_name} relation between {src} and {tgt}".format( + rel_name=r.relationship.name.lower(), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), + ) + ) if also: # recondition also to have proper subseries document names: From 76fd25a1f39093a214be8ac2e0a9ed452beb7a47 Mon Sep 17 00:00:00 2001 From: Tianyi Gao Date: Sat, 14 Mar 2026 12:19:51 +0800 Subject: [PATCH 032/188] fix: wording in id_expired_email template (#10154) --- ietf/templates/doc/draft/id_expired_email.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/templates/doc/draft/id_expired_email.txt b/ietf/templates/doc/draft/id_expired_email.txt index afbf253ee28..161146a3010 100644 --- a/ietf/templates/doc/draft/id_expired_email.txt +++ b/ietf/templates/doc/draft/id_expired_email.txt @@ -1,4 +1,4 @@ -{% autoescape off %}{{ doc.file_tag|safe }} was just expired. +{% autoescape off %}{{ doc.file_tag|safe }} just expired. This Internet-Draft is in the state "{{ state }}" in the Datatracker. From 9646edc20378e101ab48ff24253861fc5ea78fe9 Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Sun, 15 Mar 2026 02:17:40 +0800 Subject: [PATCH 033/188] feat: add author affiliation in serializer (#10549) --- ietf/api/serializers_rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index c17cbc64cee..a18dc588c4f 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -103,7 +103,7 @@ class DocumentAuthorSerializer(serializers.ModelSerializer): class Meta: model = DocumentAuthor - fields = ["person", "plain_name"] + fields = ["person", "plain_name", "affiliation"] def get_plain_name(self, document_author: DocumentAuthor) -> str: return document_author.person.plain_name() From 36fa518ec387f425b1f11f6f9040a73e8f61df30 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:02:47 -0400 Subject: [PATCH 034/188] chore(deps): bump the npm group across /dev/deploy-to-container with 5 updates (#10560) Bumps the npm group with 5 updates in the /dev/deploy-to-container directory: | Package | From | To | | --- | --- | --- | | [dockerode](https://github.com/apocas/dockerode) | `4.0.6` | `4.0.9` | | [fs-extra](https://github.com/jprichardson/node-fs-extra) | `11.3.0` | `11.3.4` | | [nanoid](https://github.com/ai/nanoid) | `5.1.5` | `5.1.7` | | [slugify](https://github.com/simov/slugify) | `1.6.6` | `1.6.8` | | [tar](https://github.com/isaacs/node-tar) | `7.4.3` | `7.5.11` | Updates `dockerode` from 4.0.6 to 4.0.9 - [Release notes](https://github.com/apocas/dockerode/releases) - [Commits](https://github.com/apocas/dockerode/compare/v4.0.6...v4.0.9) Updates `fs-extra` from 11.3.0 to 11.3.4 - [Changelog](https://github.com/jprichardson/node-fs-extra/blob/master/CHANGELOG.md) - [Commits](https://github.com/jprichardson/node-fs-extra/compare/11.3.0...11.3.4) Updates `nanoid` from 5.1.5 to 5.1.7 - [Release notes](https://github.com/ai/nanoid/releases) - [Changelog](https://github.com/ai/nanoid/blob/main/CHANGELOG.md) - [Commits](https://github.com/ai/nanoid/compare/5.1.5...5.1.7) Updates `slugify` from 1.6.6 to 1.6.8 - [Changelog](https://github.com/simov/slugify/blob/master/CHANGELOG.md) - [Commits](https://github.com/simov/slugify/compare/v1.6.6...v1.6.8) Updates `tar` from 7.4.3 to 7.5.11 - [Release notes](https://github.com/isaacs/node-tar/releases) - [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md) - [Commits](https://github.com/isaacs/node-tar/compare/v7.4.3...v7.5.11) --- updated-dependencies: - dependency-name: dockerode dependency-version: 4.0.9 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: npm - dependency-name: fs-extra dependency-version: 11.3.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: npm - dependency-name: nanoid dependency-version: 5.1.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: npm - dependency-name: slugify dependency-version: 1.6.8 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: npm - dependency-name: tar dependency-version: 7.5.11 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: npm ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dev/deploy-to-container/package-lock.json | 733 +++------------------- dev/deploy-to-container/package.json | 10 +- 2 files changed, 76 insertions(+), 667 deletions(-) diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json index 0954ec9af47..b62109f0e21 100644 --- a/dev/deploy-to-container/package-lock.json +++ b/dev/deploy-to-container/package-lock.json @@ -6,12 +6,12 @@ "": { "name": "deploy-to-container", "dependencies": { - "dockerode": "^4.0.6", - "fs-extra": "^11.3.0", - "nanoid": "5.1.5", + "dockerode": "^4.0.9", + "fs-extra": "^11.3.4", + "nanoid": "5.1.7", "nanoid-dictionary": "5.0.0", - "slugify": "1.6.6", - "tar": "^7.4.3", + "slugify": "1.6.8", + "tar": "^7.5.11", "yargs": "^17.7.2" }, "engines": { @@ -52,95 +52,6 @@ "node": ">=6" } }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" - }, - "node_modules/@isaacs/cliui/node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@isaacs/cliui/node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, "node_modules/@isaacs/fs-minipass": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", @@ -161,15 +72,6 @@ "url": "https://opencollective.com/js-sdsl" } }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "optional": true, - "engines": { - "node": ">=14" - } - }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -263,11 +165,6 @@ "safer-buffer": "~2.1.0" } }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -285,8 +182,7 @@ "type": "consulting", "url": "https://feross.org/support" } - ], - "license": "MIT" + ] }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", @@ -301,21 +197,12 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "license": "MIT", "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", "readable-stream": "^3.4.0" } }, - "node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -334,7 +221,6 @@ "url": "https://feross.org/support" } ], - "license": "MIT", "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -352,8 +238,7 @@ "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "license": "ISC" + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, "node_modules/cliui": { "version": "8.0.1", @@ -398,19 +283,6 @@ "node": ">=10.0.0" } }, - "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/debug": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", @@ -444,38 +316,31 @@ } }, "node_modules/dockerode": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz", - "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==", - "license": "Apache-2.0", + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz", + "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==", "dependencies": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", "docker-modem": "^5.0.6", "protobufjs": "^7.3.2", - "tar-fs": "~2.1.2", + "tar-fs": "^2.1.4", "uuid": "^10.0.0" }, "engines": { "node": ">= 8.0" } }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" - }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "license": "MIT", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", "dependencies": { "once": "^1.4.0" } @@ -488,32 +353,15 @@ "node": ">=6" } }, - "node_modules/foreground-child": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", - "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", - "dependencies": { - "cross-spawn": "^7.0.0", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "license": "MIT" + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "node_modules/fs-extra": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz", - "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==", - "license": "MIT", + "version": "11.3.4", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.4.tgz", + "integrity": "sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==", "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -531,27 +379,6 @@ "node": "6.* || 8.* || >= 10.*" } }, - "node_modules/glob": { - "version": "10.3.12", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", - "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^2.3.6", - "minimatch": "^9.0.1", - "minipass": "^7.0.4", - "path-scurry": "^1.10.2" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -574,8 +401,7 @@ "type": "consulting", "url": "https://feross.org/support" } - ], - "license": "BSD-3-Clause" + ] }, "node_modules/inherits": { "version": "2.0.4", @@ -590,28 +416,6 @@ "node": ">=8" } }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" - }, - "node_modules/jackspeak": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", - "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" - } - }, "node_modules/jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -633,28 +437,6 @@ "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz", "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg==" }, - "node_modules/lru-cache": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", - "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", - "engines": { - "node": "14 || >=16.14" - } - }, - "node_modules/minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", @@ -664,36 +446,20 @@ } }, "node_modules/minizlib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", - "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", "dependencies": { - "minipass": "^7.0.4", - "rimraf": "^5.0.5" + "minipass": "^7.1.2" }, "engines": { "node": ">= 18" } }, - "node_modules/mkdirp": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", - "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", - "bin": { - "mkdirp": "dist/cjs/src/bin.js" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "license": "MIT" + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, "node_modules/ms": { "version": "2.1.3", @@ -709,16 +475,15 @@ "optional": true }, "node_modules/nanoid": { - "version": "5.1.5", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz", - "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw==", + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.7.tgz", + "integrity": "sha512-ua3NDgISf6jdwezAheMOk4mbE1LXjm1DfMUDMuJf4AqxLFK3ccGpgWizwa5YV7Yz9EpXwEaWoRXSb/BnV0t5dQ==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/ai" } ], - "license": "MIT", "bin": { "nanoid": "bin/nanoid.js" }, @@ -736,34 +501,10 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "license": "ISC", "dependencies": { "wrappy": "1" } }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-scurry": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", - "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", - "dependencies": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/protobufjs": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", @@ -788,10 +529,9 @@ } }, "node_modules/pump": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", - "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", - "license": "MIT", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -818,23 +558,6 @@ "node": ">=0.10.0" } }, - "node_modules/rimraf": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", - "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", - "dependencies": { - "glob": "^10.3.7" - }, - "bin": { - "rimraf": "dist/esm/bin.mjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -860,40 +583,10 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "engines": { - "node": ">=8" - } - }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/slugify": { - "version": "1.6.6", - "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", - "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==", + "version": "1.6.8", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.8.tgz", + "integrity": "sha512-HVk9X1E0gz3mSpoi60h/saazLKXKaZThMLU3u/aNwoYn8/xQyX2MGxL0ui2eaokkD7tF+Zo+cKTHUbe1mmmGzA==", "engines": { "node": ">=8.0.0" } @@ -942,20 +635,6 @@ "node": ">=8" } }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -967,28 +646,15 @@ "node": ">=8" } }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/tar": { - "version": "7.4.3", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", - "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "version": "7.5.11", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz", + "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", - "minizlib": "^3.0.1", - "mkdirp": "^3.0.1", + "minizlib": "^3.1.0", "yallist": "^5.0.0" }, "engines": { @@ -996,10 +662,9 @@ } }, "node_modules/tar-fs": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz", - "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==", - "license": "MIT", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -1011,7 +676,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "license": "MIT", "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -1067,20 +731,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -1097,28 +747,10 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "license": "ISC" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/y18n": { "version": "5.0.8", @@ -1188,64 +820,6 @@ "yargs": "^17.7.2" } }, - "@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "requires": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" - }, - "ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==" - }, - "emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" - }, - "string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "requires": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - } - }, - "strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "requires": { - "ansi-regex": "^6.0.1" - } - }, - "wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "requires": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - } - } - } - }, "@isaacs/fs-minipass": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", @@ -1259,12 +833,6 @@ "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==" }, - "@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "optional": true - }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -1348,11 +916,6 @@ "safer-buffer": "~2.1.0" } }, - "balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1376,14 +939,6 @@ "readable-stream": "^3.4.0" } }, - "brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "requires": { - "balanced-match": "^1.0.0" - } - }, "buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -1437,16 +992,6 @@ "nan": "^2.19.0" } }, - "cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "requires": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - } - }, "debug": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", @@ -1467,33 +1012,28 @@ } }, "dockerode": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz", - "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==", + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz", + "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==", "requires": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", "docker-modem": "^5.0.6", "protobufjs": "^7.3.2", - "tar-fs": "~2.1.2", + "tar-fs": "^2.1.4", "uuid": "^10.0.0" } }, - "eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" - }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", "requires": { "once": "^1.4.0" } @@ -1503,24 +1043,15 @@ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, - "foreground-child": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", - "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", - "requires": { - "cross-spawn": "^7.0.0", - "signal-exit": "^4.0.1" - } - }, "fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "fs-extra": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz", - "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==", + "version": "11.3.4", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.4.tgz", + "integrity": "sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==", "requires": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -1532,18 +1063,6 @@ "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, - "glob": { - "version": "10.3.12", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", - "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", - "requires": { - "foreground-child": "^3.1.0", - "jackspeak": "^2.3.6", - "minimatch": "^9.0.1", - "minipass": "^7.0.4", - "path-scurry": "^1.10.2" - } - }, "graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -1564,20 +1083,6 @@ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, - "isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" - }, - "jackspeak": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", - "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", - "requires": { - "@isaacs/cliui": "^8.0.2", - "@pkgjs/parseargs": "^0.11.0" - } - }, "jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -1597,38 +1102,19 @@ "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz", "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg==" }, - "lru-cache": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", - "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==" - }, - "minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "requires": { - "brace-expansion": "^2.0.1" - } - }, "minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==" }, "minizlib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", - "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", "requires": { - "minipass": "^7.0.4", - "rimraf": "^5.0.5" + "minipass": "^7.1.2" } }, - "mkdirp": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", - "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==" - }, "mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", @@ -1646,9 +1132,9 @@ "optional": true }, "nanoid": { - "version": "5.1.5", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz", - "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw==" + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.7.tgz", + "integrity": "sha512-ua3NDgISf6jdwezAheMOk4mbE1LXjm1DfMUDMuJf4AqxLFK3ccGpgWizwa5YV7Yz9EpXwEaWoRXSb/BnV0t5dQ==" }, "nanoid-dictionary": { "version": "5.0.0", @@ -1663,20 +1149,6 @@ "wrappy": "1" } }, - "path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" - }, - "path-scurry": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", - "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", - "requires": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - } - }, "protobufjs": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", @@ -1697,9 +1169,9 @@ } }, "pump": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", - "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -1720,14 +1192,6 @@ "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" }, - "rimraf": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", - "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", - "requires": { - "glob": "^10.3.7" - } - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1738,28 +1202,10 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "requires": { - "shebang-regex": "^3.0.0" - } - }, - "shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" - }, - "signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" - }, "slugify": { - "version": "1.6.6", - "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", - "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==" + "version": "1.6.8", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.8.tgz", + "integrity": "sha512-HVk9X1E0gz3mSpoi60h/saazLKXKaZThMLU3u/aNwoYn8/xQyX2MGxL0ui2eaokkD7tF+Zo+cKTHUbe1mmmGzA==" }, "split-ca": { "version": "1.0.1", @@ -1795,16 +1241,6 @@ "strip-ansi": "^6.0.1" } }, - "string-width-cjs": { - "version": "npm:string-width@4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -1813,24 +1249,15 @@ "ansi-regex": "^5.0.1" } }, - "strip-ansi-cjs": { - "version": "npm:strip-ansi@6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - }, "tar": { - "version": "7.4.3", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", - "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "version": "7.5.11", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz", + "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==", "requires": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", - "minizlib": "^3.0.1", - "mkdirp": "^3.0.1", + "minizlib": "^3.1.0", "yallist": "^5.0.0" }, "dependencies": { @@ -1842,9 +1269,9 @@ } }, "tar-fs": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz", - "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", "requires": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -1889,14 +1316,6 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==" }, - "which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "requires": { - "isexe": "^2.0.0" - } - }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -1907,16 +1326,6 @@ "strip-ansi": "^6.0.0" } }, - "wrap-ansi-cjs": { - "version": "npm:wrap-ansi@7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json index 09716c3094a..1c95a4540ca 100644 --- a/dev/deploy-to-container/package.json +++ b/dev/deploy-to-container/package.json @@ -2,12 +2,12 @@ "name": "deploy-to-container", "type": "module", "dependencies": { - "dockerode": "^4.0.6", - "fs-extra": "^11.3.0", - "nanoid": "5.1.5", + "dockerode": "^4.0.9", + "fs-extra": "^11.3.4", + "nanoid": "5.1.7", "nanoid-dictionary": "5.0.0", - "slugify": "1.6.6", - "tar": "^7.4.3", + "slugify": "1.6.8", + "tar": "^7.5.11", "yargs": "^17.7.2" }, "engines": { From dcce2df0300879078690a2fbd3522602d467cf38 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Fri, 20 Mar 2026 12:03:18 +0900 Subject: [PATCH 035/188] feat: add attendance summary and pie chart to meeting attendees page (#10481) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add attendance summary and pie chart to meeting attendees page For IETF meetings ≥ 118, the attendees proceedings page now shows an Onsite / Remote / Total summary row matching the counts displayed on registration.ietf.org, together with a "View chart" button that opens a Bootstrap modal containing a Highcharts pie chart. * Split out attendees-chart.js --- ietf/meeting/tests_views.py | 13 +++++ ietf/meeting/views.py | 33 +++++++++-- ietf/static/js/attendees-chart.js | 58 +++++++++++++++++++ .../meeting/proceedings_attendees.html | 53 ++++++++++++++++- package.json | 1 + 5 files changed, 151 insertions(+), 7 deletions(-) create mode 100644 ietf/static/js/attendees-chart.js diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 168999d0aa7..258ffe554c2 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -9007,6 +9007,8 @@ def test_proceedings_attendees(self): - assert onsite checkedin=True appears, not onsite checkedin=False - assert remote attended appears, not remote not attended - prefer onsite checkedin=True to remote attended when same person has both + - summary stats row shows correct counts + - chart data JSON is embedded with correct values """ m = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number="118") @@ -9028,6 +9030,17 @@ def test_proceedings_attendees(self): text = q('#id_attendees tbody tr').text().replace('\n', ' ') self.assertEqual(text, f"A Person {areg.affiliation} {areg.country_code} onsite C Person {creg.affiliation} {creg.country_code} remote") + # Summary stats row: Onsite / Remote / Total (matches registration.ietf.org) + self.assertContains(response, 'Onsite:') + self.assertContains(response, 'Remote:') + self.assertContains(response, 'Total:') + self.assertContains(response, '1') # onsite and remote + self.assertContains(response, '2') # total + + # Chart data embedded in page + chart_json = json.loads(q('#attendees-chart-data').text()) + self.assertEqual(chart_json['type'], [['Onsite', 1], ['Remote', 1]]) + def test_proceedings_overview(self): '''Test proceedings IETF Overview page. Note: old meetings aren't supported so need to add a new meeting then test. diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 731dfad88fb..67a81305b49 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -109,7 +109,7 @@ from ietf.meeting.utils import get_activity_stats, post_process, create_recording, delete_recording from ietf.meeting.utils import participants_for_meeting, generate_bluesheet, bluesheet_data, save_bluesheet from ietf.message.utils import infer_message -from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName +from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName, CountryName from ietf.utils import markdown from ietf.utils.decorators import require_api_key from ietf.utils.hedgedoc import Note, NoteError @@ -4812,15 +4812,36 @@ def proceedings_attendees(request, num=None): template = None registrations = None + stats = None + chart_data = None + if int(meeting.number) >= 118: checked_in, attended = participants_for_meeting(meeting) regs = list(Registration.objects.onsite().filter(meeting__number=num, checkedin=True)) - - for reg in Registration.objects.remote().filter(meeting__number=num).select_related('person'): - if reg.person.pk in attended and reg.person.pk not in checked_in: - regs.append(reg) + onsite_count = len(regs) + regs += [ + reg + for reg in Registration.objects.remote().filter(meeting__number=num).select_related('person') + if reg.person.pk in attended and reg.person.pk not in checked_in + ] + remote_count = len(regs) - onsite_count registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name)) + + country_codes = [r.country_code for r in registrations if r.country_code] + stats = { + 'total': onsite_count + remote_count, + 'onsite': onsite_count, + 'remote': remote_count, + } + + code_to_name = dict(CountryName.objects.values_list('slug', 'name')) + country_counts = Counter(code_to_name.get(c, c) for c in country_codes).most_common() + + chart_data = { + 'type': [['Onsite', onsite_count], ['Remote', remote_count]], + 'countries': country_counts, + } else: overview_template = "/meeting/proceedings/%s/attendees.html" % meeting.number try: @@ -4832,6 +4853,8 @@ def proceedings_attendees(request, num=None): 'meeting': meeting, 'registrations': registrations, 'template': template, + 'stats': stats, + 'chart_data': chart_data, }) def proceedings_overview(request, num=None): diff --git a/ietf/static/js/attendees-chart.js b/ietf/static/js/attendees-chart.js new file mode 100644 index 00000000000..fed3b1289cb --- /dev/null +++ b/ietf/static/js/attendees-chart.js @@ -0,0 +1,58 @@ +(function () { + var raw = document.getElementById('attendees-chart-data'); + if (!raw) return; + var chartData = JSON.parse(raw.textContent); + var chart = null; + var currentBreakdown = 'type'; + + // Override the global transparent background set by highcharts.js so the + // export menu and fullscreen view use the page background color. + var container = document.getElementById('attendees-pie-chart'); + var bodyBg = getComputedStyle(document.body).backgroundColor; + container.style.setProperty('--highcharts-background-color', bodyBg); + + function renderChart(breakdown) { + var seriesData = chartData[breakdown].map(function (item) { + return { name: item[0], y: item[1] }; + }); + if (chart) chart.destroy(); + chart = Highcharts.chart(container, { + chart: { type: 'pie', height: 400 }, + title: { text: null }, + tooltip: { pointFormat: '{point.name}: {point.y} ({point.percentage:.1f}%)' }, + plotOptions: { + pie: { + dataLabels: { + enabled: true, + format: '{point.name}
{point.y} ({point.percentage:.1f}%)', + }, + showInLegend: false, + } + }, + series: [{ name: 'Attendees', data: seriesData }], + }); + } + + var modal = document.getElementById('attendees-chart-modal'); + + // Render (or re-render) the chart each time the modal becomes fully visible, + // so Highcharts can measure the container dimensions correctly. + modal.addEventListener('shown.bs.modal', function () { + renderChart(currentBreakdown); + }); + + // Release the chart when the modal closes to avoid stale renders. + modal.addEventListener('hidden.bs.modal', function () { + if (chart) { + chart.destroy(); + chart = null; + } + }); + + document.querySelectorAll('[name="attendees-breakdown"]').forEach(function (radio) { + radio.addEventListener('change', function () { + currentBreakdown = this.value; + renderChart(currentBreakdown); + }); + }); +})(); diff --git a/ietf/templates/meeting/proceedings_attendees.html b/ietf/templates/meeting/proceedings_attendees.html index 390ce00cadd..0c59d4ab155 100644 --- a/ietf/templates/meeting/proceedings_attendees.html +++ b/ietf/templates/meeting/proceedings_attendees.html @@ -3,6 +3,7 @@ {% load origin markup_tags static %} {% block pagehead %} + {% if chart_data %}{% endif %} {% endblock %} {% block title %}IETF {{ meeting.number }} proceedings{% endblock %} {% block content %} @@ -14,8 +15,52 @@

Attendee list of IETF {{ meeting.number }} meeting

- + + {% if chart_data %} +
+
+
Onsite: {{ stats.onsite }}
+
Remote: {{ stats.remote }}
+
Total: {{ stats.total }}
+
+ +
+ + + + {{ chart_data|json_script:"attendees-chart-data" }} + {% endif %}{# chart_data #} + {% if template %} + {{template|safe}} {% else %} @@ -44,4 +89,8 @@

Attendee list of IETF {{ meeting.number }} meeting

{% endblock %} {% block js %} -{% endblock %} \ No newline at end of file + {% if chart_data %} + + + {% endif %} +{% endblock %} diff --git a/package.json b/package.json index fec29275b47..bb71250c4be 100644 --- a/package.json +++ b/package.json @@ -112,6 +112,7 @@ "ietf/static/images/irtf-logo-white.svg", "ietf/static/images/irtf-logo.svg", "ietf/static/js/add_session_recordings.js", + "ietf/static/js/attendees-chart.js", "ietf/static/js/agenda_filter.js", "ietf/static/js/agenda_materials.js", "ietf/static/js/announcement.js", From 2c29cbaad91a7c076643167e1ffc056975b2c97e Mon Sep 17 00:00:00 2001 From: Tianyi Gao Date: Fri, 20 Mar 2026 11:45:09 +0800 Subject: [PATCH 036/188] feat: add parent section in team about (#9148) (#10551) fix: remove empty for area/parent on all groups --- ietf/group/tests_info.py | 19 +++++++++++++++++++ ietf/templates/group/group_about.html | 11 +++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 34f8500854b..3f24e2e3d6b 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -543,6 +543,25 @@ def verify_can_edit_group(url, group, username): for username in list(set(interesting_users)-set(can_edit[group.type_id])): verify_cannot_edit_group(url, group, username) + def test_group_about_team_parent(self): + """Team about page should show parent when parent is not an area""" + GroupFactory(type_id='team', parent=GroupFactory(type_id='area', acronym='gen')) + GroupFactory(type_id='team', parent=GroupFactory(type_id='ietf', acronym='iab')) + GroupFactory(type_id='team', parent=None) + + for team in Group.objects.filter(type='team').select_related('parent'): + url = urlreverse('ietf.group.views.group_about', kwargs=dict(acronym=team.acronym)) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + if team.parent and team.parent.type_id != 'area': + self.assertContains(r, 'Parent') + self.assertContains(r, team.parent.acronym) + elif team.parent and team.parent.type_id == 'area': + self.assertContains(r, team.parent.name) + self.assertNotContains(r, '>Parent<') + else: + self.assertNotContains(r, '>Parent<') + def test_group_about_personnel(self): """Correct personnel should appear on the group About page""" group = GroupFactory() diff --git a/ietf/templates/group/group_about.html b/ietf/templates/group/group_about.html index cbc2e11536f..6d1843383ca 100644 --- a/ietf/templates/group/group_about.html +++ b/ietf/templates/group/group_about.html @@ -51,10 +51,13 @@ {{ group.parent.name }} ({{ group.parent.acronym }}) - {% else %} - + {% elif group.parent and group.type_id == "team" %} + - + {% endif %} @@ -444,4 +447,4 @@

group_stats("{% url 'ietf.group.views.group_stats_data' %}", ".chart"); }); -{% endblock %} \ No newline at end of file +{% endblock %} From abab6373f5f465bdcc052f45c5def0710f360dc7 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 19 Mar 2026 23:02:40 -0500 Subject: [PATCH 037/188] fix: reduce db churn and log noise for rebuilding references (#10563) * fix: (wip) reduce db churn and log noise for rebuilding references * fix: typo in log message * fix: typo in log message Co-authored-by: Jennifer Richards --------- Co-authored-by: Jennifer Richards --- ietf/doc/utils.py | 78 ++++++++++++++++++++++++++++++-------------- ietf/submit/utils.py | 5 +-- 2 files changed, 54 insertions(+), 29 deletions(-) diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 396b3fcfa47..8cbe5e8f3ec 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -4,6 +4,7 @@ import datetime import io +import json import math import os import re @@ -954,58 +955,78 @@ def rebuild_reference_relations(doc, filenames): filenames should be a dict mapping file ext (i.e., type) to the full path of each file. """ if doc.type.slug not in ["draft", "rfc"]: + log.log(f"rebuild_reference_relations called for non draft/rfc doc {doc.name}") return None - - log.log(f"Rebuilding reference relations for {doc.name}") - # try XML first - if "xml" in filenames: - refs = XMLDraft(filenames["xml"]).get_refs() - elif "txt" in filenames: - filename = filenames["txt"] - try: - refs = draft.PlaintextDraft.from_file(filename).get_refs() - except IOError as e: - return {"errors": [f"{e.strerror}: {filename}"]} - else: + + if "xml" not in filenames and "txt" not in filenames: + log.log(f"rebuild_reference_relations error: no file available for {doc.name}") return { "errors": [ "No file available for rebuilding reference relations. Need XML or plaintext." ] } - - doc.relateddocument_set.filter( + else: + try: + # try XML first + if "xml" in filenames: + refs = XMLDraft(filenames["xml"]).get_refs() + elif "txt" in filenames: + filename = filenames["txt"] + refs = draft.PlaintextDraft.from_file(filename).get_refs() + except (IOError, UnicodeDecodeError) as e: + log.log(f"rebuild_reference_relations error: On {doc.name}: {e}") + return {"errors": [f"{e}: {filename}"]} + + before = set(doc.relateddocument_set.filter( relationship__slug__in=["refnorm", "refinfo", "refold", "refunk"] - ).delete() + ).values_list("relationship__slug","target__name")) warnings = [] errors = [] unfound = set() + intended = set() + names = [ref for ref in refs] + names.extend([ref[:-3] for ref in refs if re.match(r"^draft-.*-\d{2}$", ref)]) + queryset = Document.objects.filter(name__in=names) for ref, refType in refs.items(): - refdoc = Document.objects.filter(name=ref) - if not refdoc and re.match(r"^draft-.*-\d{2}$", ref): - refdoc = Document.objects.filter(name=ref[:-3]) + refdoc = queryset.filter(name=ref) + if not refdoc.exists() and re.match(r"^draft-.*-\d{2}$", ref): + refdoc = queryset.filter(name=ref[:-3]) count = refdoc.count() if count == 0: unfound.add("%s" % ref) continue elif count > 1: + log.unreachable("2026-3-16") # This branch is holdover from DocAlias errors.append("Too many Document objects found for %s" % ref) else: # Don't add references to ourself if doc != refdoc[0]: - RelatedDocument.objects.get_or_create( - source=doc, - target=refdoc[0], - relationship=DocRelationshipName.objects.get( - slug="ref%s" % refType - ), - ) + intended.add((f"ref{refType}", refdoc[0].name)) + if unfound: warnings.append( "There were %d references with no matching Document" % len(unfound) ) + if intended != before: + for slug, name in before-intended: + doc.relateddocument_set.filter(target__name=name, relationship_id=slug).delete() + for slug, name in intended-before: + doc.relateddocument_set.create( + target=queryset.get(name=name), + relationship_id=slug + ) + after = set(doc.relateddocument_set.filter( + relationship__slug__in=["refnorm", "refinfo", "refold", "refunk"] + ).values_list("relationship__slug","target__name")) + if after != intended: + errors.append("Attempted changed didn't achieve intended results") + changed_references = True + else: + changed_references = False + ret = {} if errors: ret["errors"] = errors @@ -1014,6 +1035,13 @@ def rebuild_reference_relations(doc, filenames): if unfound: ret["unfound"] = list(unfound) + logmsg = f"rebuild_reference_relations for {doc.name}: " + logmsg += "changed references" if changed_references else "references unchanged" + if ret: + logmsg += f" {json.dumps(ret)}" + + log.log(logmsg) + return ret def set_replaces_for_document(request, doc, new_replaces, by, email_subject, comment=""): diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 9a7c358a6d3..7e3106f723c 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -395,10 +395,7 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc): log.log(f"{submission.name}: updated state and info") - trouble = rebuild_reference_relations(draft, find_submission_filenames(draft)) - if trouble: - log.log('Rebuild_reference_relations trouble: %s'%trouble) - log.log(f"{submission.name}: rebuilt reference relations") + rebuild_reference_relations(draft, find_submission_filenames(draft)) if draft.stream_id == "ietf" and draft.group.type_id == "wg" and draft.rev == "00": # automatically set state "WG Document" From b08945aaf4618613f668bb5c231533f709bea4d4 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 20 Mar 2026 05:17:10 -0300 Subject: [PATCH 038/188] fix: maintain column count in HTML template (#10593) --- ietf/templates/group/group_about.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ietf/templates/group/group_about.html b/ietf/templates/group/group_about.html index 6d1843383ca..0a8b9194f28 100644 --- a/ietf/templates/group/group_about.html +++ b/ietf/templates/group/group_about.html @@ -58,6 +58,10 @@ {{ group.parent.name }} ({{ group.parent.acronym }}) + {% else %} +

+ + {% endif %} From d39317b070a7af5db4f48edaf0e7f03fd0a29680 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 23 Mar 2026 12:33:23 -0300 Subject: [PATCH 039/188] feat: update typesense search index on rfc pub/update (#10575) * chore: typesense API config for k8s * feat: DocumentInfo.pub_datetime() helper * chore(deps): install typesense library * feat: searchindex (typesense) util module * feat: sanitize abstract * feat: add (sanitized) content * style: ruff ruff on doc/tasks.py * feat: search index update task * chore: call the update task * refactor: better settings management * ci: update prod settings * chore: typing * test: searchindex tests * test: searchindex task test * style: ruff ruff * chore: drop type hints to fix mypy errors * test: fix tests * test: improve coverage * fix: handle missing content blob correctly --- ietf/api/serializers_rpc.py | 5 +- ietf/api/tests_serializers_rpc.py | 18 +++- ietf/api/tests_views_rpc.py | 44 ++++++--- ietf/api/views_rpc.py | 3 +- ietf/doc/models.py | 17 ++-- ietf/doc/tasks.py | 46 +++++++-- ietf/doc/tests_tasks.py | 64 ++++++++++-- ietf/utils/searchindex.py | 155 ++++++++++++++++++++++++++++++ ietf/utils/tests_searchindex.py | 128 ++++++++++++++++++++++++ k8s/settings_local.py | 20 ++-- requirements.txt | 1 + 11 files changed, 451 insertions(+), 50 deletions(-) create mode 100644 ietf/utils/searchindex.py create mode 100644 ietf/utils/tests_searchindex.py diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index a18dc588c4f..701f05eece8 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2025, All Rights Reserved +# Copyright The IETF Trust 2025-2026, All Rights Reserved import datetime from pathlib import Path from typing import Literal, Optional @@ -20,6 +20,7 @@ RfcAuthor, ) from ietf.doc.serializers import RfcAuthorSerializer +from ietf.doc.tasks import update_rfc_searchindex_task from ietf.doc.utils import ( default_consensus, prettify_std_name, @@ -682,6 +683,8 @@ def update(self, instance, validated_data): stale_subseries_relations.delete() if len(rfc_events) > 0: rfc.save_with_history(rfc_events) + + update_rfc_searchindex_task.delay(rfc.rfc_number) return rfc diff --git a/ietf/api/tests_serializers_rpc.py b/ietf/api/tests_serializers_rpc.py index 1babb4c30f9..ed326be451d 100644 --- a/ietf/api/tests_serializers_rpc.py +++ b/ietf/api/tests_serializers_rpc.py @@ -1,4 +1,6 @@ # Copyright The IETF Trust 2026, All Rights Reserved +from unittest import mock + from django.utils import timezone from ietf.utils.test_utils import TestCase @@ -32,7 +34,8 @@ def test_create(self): with self.assertRaises(RuntimeError, msg="serializer does not allow create()"): serializer.save() - def test_update(self): + @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") + def test_update(self, mock_update_searchindex_task): rfc = WgRfcFactory(pages=10) serializer = EditableRfcSerializer( instance=rfc, @@ -56,6 +59,11 @@ def test_update(self): ) self.assertTrue(serializer.is_valid()) result = serializer.save() + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) result.refresh_from_db() self.assertEqual(result.title, "Yadda yadda yadda") self.assertEqual( @@ -84,7 +92,8 @@ def test_update(self): [Document.objects.get(name="fyi999")], ) - def test_partial_update(self): + @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") + def test_partial_update(self, mock_update_searchindex_task): # We could test other permutations of fields, but authors is a partial update # we know we are going to use, so verifying that one in particular. rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan") @@ -104,6 +113,11 @@ def test_partial_update(self): ) self.assertTrue(serializer.is_valid()) result = serializer.save() + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) result.refresh_from_db() self.assertEqual(rfc.title, "padawan") self.assertEqual( diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 7ab8778d287..a679e747891 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -196,7 +196,8 @@ def test_notify_rfc_published(self, mock_task_delay): self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) - def test_upload_rfc_files(self): + @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task") + def test_upload_rfc_files(self, mock_update_searchindex_task): def _valid_post_data(): """Generate a valid post data dict @@ -217,14 +218,7 @@ def _valid_post_data(): } url = urlreverse("ietf.api.purple_api.upload_rfc_files") - unused_rfc_number = ( - Document.objects.filter(rfc_number__isnull=False).aggregate( - unused_rfc_number=Max("rfc_number") + 1 - )["unused_rfc_number"] - or 10000 - ) - - rfc = WgRfcFactory(rfc_number=unused_rfc_number) + rfc = WgRfcFactory() assert isinstance(rfc, Document), "WgRfcFactory should generate a Document" with TemporaryDirectory() as rfc_dir: settings.RFC_PATH = rfc_dir # affects overridden settings @@ -236,15 +230,17 @@ def _valid_post_data(): # no api key r = self.client.post(url, _valid_post_data(), format="multipart") self.assertEqual(r.status_code, 403) + self.assertFalse(mock_update_searchindex_task.delay.called) # invalid RFC r = self.client.post( url, - _valid_post_data() | {"rfc": unused_rfc_number + 1}, + _valid_post_data() | {"rfc": rfc.rfc_number + 10}, format="multipart", headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) # empty files r = self.client.post( @@ -263,6 +259,7 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) # bad file type r = self.client.post( @@ -276,9 +273,10 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) # Put a file in the way. Post should fail because replace = False - file_in_the_way = (rfc_path / f"rfc{unused_rfc_number}.txt") + file_in_the_way = (rfc_path / f"{rfc.name}.txt") file_in_the_way.touch() r = self.client.post( url, @@ -287,11 +285,12 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) file_in_the_way.unlink() # Put a blob in the way. Post should fail because replace = False blob_in_the_way = Blob.objects.create( - bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b"" + bucket="rfc", name=f"txt/{rfc.name}.txt", content=b"" ) r = self.client.post( url, @@ -300,6 +299,7 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) blob_in_the_way.delete() # valid post @@ -310,8 +310,13 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 200) + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) for extension in ["xml", "txt", "html", "pdf", "json"]: - filename = f"rfc{unused_rfc_number}.{extension}" + filename = f"{rfc.name}.{extension}" self.assertEqual( (rfc_path / filename) .read_text(), @@ -328,7 +333,7 @@ def _valid_post_data(): f"{extension} blob should contain the expected content", ) # special case for notprepped - notprepped_fn = f"rfc{unused_rfc_number}.notprepped.xml" + notprepped_fn = f"{rfc.name}.notprepped.xml" self.assertEqual( ( rfc_path / "prerelease" / notprepped_fn @@ -347,6 +352,7 @@ def _valid_post_data(): ) # re-post with replace = False should now fail + mock_update_searchindex_task.reset_mock() r = self.client.post( url, _valid_post_data(), @@ -354,7 +360,8 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 409) # conflict - + self.assertFalse(mock_update_searchindex_task.delay.called) + # re-post with replace = True should succeed r = self.client.post( url, @@ -362,7 +369,12 @@ def _valid_post_data(): format="multipart", headers={"X-Api-Key": "valid-token"}, ) - self.assertEqual(r.status_code, 200) # conflict + self.assertEqual(r.status_code, 200) + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) @mock.patch("ietf.api.views_rpc.create_rfc_index_task") diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index c7ae6990056..cb6a59a1674 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -38,7 +38,7 @@ from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent from ietf.doc.serializers import RfcAuthorSerializer from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage -from ietf.doc.tasks import signal_update_rfc_metadata_task +from ietf.doc.tasks import signal_update_rfc_metadata_task, update_rfc_searchindex_task from ietf.person.models import Email, Person from ietf.sync.tasks import create_rfc_index_task @@ -516,6 +516,7 @@ def post(self, request): destination.parent.mkdir() shutil.move(ftm, destination) + update_rfc_searchindex_task.delay(rfc.rfc_number) return Response(NotificationAckSerializer().data) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 7b23a62c45b..972f0a34e8e 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1285,11 +1285,8 @@ def submission(self): s = s.first() return s - def pub_date(self): - """Get the publication date for this document - - This is the rfc publication date for RFCs, and the new-revision date for other documents. - """ + def pub_datetime(self): + """Get the publication datetime of this document""" if self.type_id == "rfc": # As of Sept 2022, in ietf.sync.rfceditor.update_docs_from_rfc_index() `published_rfc` events are # created with a timestamp whose date *in the PST8PDT timezone* is the official publication date @@ -1297,7 +1294,15 @@ def pub_date(self): event = self.latest_event(type='published_rfc') else: event = self.latest_event(type='new_revision') - return event.time.astimezone(RPC_TZINFO).date() if event else None + return event.time.astimezone(RPC_TZINFO) if event else None + + def pub_date(self): + """Get the publication date for this document + + This is the rfc publication date for RFCs, and the new-revision date for other documents. + """ + pub_datetime = self.pub_datetime() + return None if pub_datetime is None else pub_datetime.date() def is_dochistory(self): return False diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index 90f4c80af57..a38cd5eb5cf 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -3,6 +3,7 @@ # Celery task definitions # import datetime + import debug # pyflakes:ignore from celery import shared_task @@ -11,7 +12,7 @@ from django.conf import settings from django.utils import timezone -from ietf.utils import log +from ietf.utils import log, searchindex from ietf.utils.timezone import datetime_today from .expire import ( @@ -77,17 +78,19 @@ def expire_last_calls_task(): try: expire_last_call(doc) except Exception: - log.log(f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})") + log.log( + f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})" + ) else: log.log(f"Expired last call for {doc.file_tag()} (id={doc.pk})") -@shared_task +@shared_task def generate_idnits2_rfc_status_task(): outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status" blob = generate_idnits2_rfc_status() try: - outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE + outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE except Exception as e: log.log(f"failed to write idnits2-rfc-status: {e}") @@ -97,7 +100,7 @@ def generate_idnits2_rfcs_obsoleted_task(): outpath = Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted" blob = generate_idnits2_rfcs_obsoleted() try: - outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE + outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE except Exception as e: log.log(f"failed to write idnits2-rfcs-obsoleted: {e}") @@ -105,7 +108,7 @@ def generate_idnits2_rfcs_obsoleted_task(): @shared_task def generate_draft_bibxml_files_task(days=7, process_all=False): """Generate bibxml files for recently updated docs - + If process_all is False (the default), processes only docs with new revisions in the last specified number of days. """ @@ -117,7 +120,9 @@ def generate_draft_bibxml_files_task(days=7, process_all=False): doc__type_id="draft", ).order_by("time") if not process_all: - doc_events = doc_events.filter(time__gte=timezone.now() - datetime.timedelta(days=days)) + doc_events = doc_events.filter( + time__gte=timezone.now() - datetime.timedelta(days=days) + ) for event in doc_events: try: update_or_create_draft_bibxml_file(event.doc, event.rev) @@ -132,6 +137,7 @@ def investigate_fragment_task(name_fragment: str): "results": investigate_fragment(name_fragment), } + @shared_task def rebuild_reference_relations_task(doc_names: list[str]): log.log(f"Task: Rebuilding reference relations for {doc_names}") @@ -157,6 +163,32 @@ def rebuild_reference_relations_task(doc_names: list[str]): def fixup_bofreq_timestamps_task(): # pragma: nocover fixup_bofreq_timestamps() + @shared_task def signal_update_rfc_metadata_task(rfc_number_list=()): signal_update_rfc_metadata(rfc_number_list) + + +@shared_task(bind=True) +def update_rfc_searchindex_task(self, rfc_number: int): + """Update the search index for one RFC""" + if not searchindex.enabled(): + log.log("Search indexing is not enabled, skipping") + return + + rfc = Document.objects.filter(type_id="rfc", rfc_number=rfc_number).first() + if rfc is None: + log.log( + f"ERROR: Document for rfc{rfc_number} not found, not updating search index" + ) + return + try: + searchindex.update_or_create_rfc_entry(rfc) + except Exception as err: + log.log(f"Search index update for {rfc.name} failed ({err})") + if isinstance(err, searchindex.RETRYABLE_ERROR_CLASSES): + searchindex_settings = searchindex.get_settings() + self.retry( + countdown=searchindex_settings["TASK_RETRY_DELAY"], + max_retries=searchindex_settings["TASK_MAX_RETRIES"], + ) diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 29689cd5964..728d21f131f 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -1,18 +1,20 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved -import debug # pyflakes:ignore import datetime from unittest import mock from pathlib import Path +from celery.exceptions import Retry from django.conf import settings +from django.test.utils import override_settings from django.utils import timezone +from typesense import exceptions as typesense_exceptions from ietf.utils.test_utils import TestCase from ietf.utils.timezone import datetime_today -from .factories import DocumentFactory, NewRevisionDocEventFactory +from .factories import DocumentFactory, NewRevisionDocEventFactory, WgRfcFactory from .models import Document, NewRevisionDocEvent from .tasks import ( expire_ids_task, @@ -22,8 +24,10 @@ generate_idnits2_rfc_status_task, investigate_fragment_task, notify_expirations_task, + update_rfc_searchindex_task, ) + class TaskTests(TestCase): @mock.patch("ietf.doc.tasks.in_draft_expire_freeze") @mock.patch("ietf.doc.tasks.get_expired_drafts") @@ -87,7 +91,7 @@ def test_expire_last_calls_task(self, mock_get_expired, mock_expire): self.assertEqual(mock_expire.call_args_list[0], mock.call(docs[0])) self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1])) self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2])) - + # Check that it runs even if exceptions occur mock_get_expired.reset_mock() mock_expire.reset_mock() @@ -111,9 +115,40 @@ def test_investigate_fragment_task(self): retval, {"name_fragment": "some fragment", "results": investigation_results} ) + @mock.patch("ietf.doc.tasks.searchindex.update_or_create_rfc_entry") + @mock.patch("ietf.doc.tasks.searchindex.enabled") + def test_update_rfc_searchindex_task( + self, mock_searchindex_enabled, mock_create_entry + ): + mock_searchindex_enabled.return_value = False + + self.assertFalse(Document.objects.filter(rfc_number=5073).exists()) + rfc = WgRfcFactory() + update_rfc_searchindex_task(rfc_number=5073) + self.assertFalse(mock_create_entry.called) + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + self.assertFalse(mock_create_entry.called) + + mock_searchindex_enabled.return_value = True + update_rfc_searchindex_task(rfc_number=5073) + self.assertFalse(mock_create_entry.called) + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + self.assertTrue(mock_create_entry.called) + + with override_settings(SEARCHINDEX_CONFIG={"TASK_MAX_RETRIES": 0}): + # Try a non-retryable error (there are others) + mock_create_entry.side_effect = typesense_exceptions.RequestMalformed + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) # no retry + # Now what should be a retryable error + mock_create_entry.side_effect = typesense_exceptions.Timeout + with self.assertRaises(Retry): + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + class Idnits2SupportTests(TestCase): - settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR'] + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + [ + "DERIVED_DIR" + ] @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted") def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate): @@ -151,7 +186,9 @@ def setUp(self): ) # a couple that should always be ignored NewRevisionDocEventFactory( - time=now - datetime.timedelta(days=6), rev="09", doc__type_id="rfc" # not a draft + time=now - datetime.timedelta(days=6), + rev="09", + doc__type_id="rfc", # not a draft ) NewRevisionDocEventFactory( type="changed_document", # not a "new_revision" type @@ -164,7 +201,9 @@ def setUp(self): @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_bibxml_files_for_all_drafts_task(self, mock_create, mock_ensure_path): + def test_generate_bibxml_files_for_all_drafts_task( + self, mock_create, mock_ensure_path + ): generate_draft_bibxml_files_task(process_all=True) self.assertTrue(mock_ensure_path.called) self.assertCountEqual( @@ -193,12 +232,15 @@ def test_generate_bibxml_files_for_all_drafts_task(self, mock_create, mock_ensur @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_ensure_path): + def test_generate_bibxml_files_for_recent_drafts_task( + self, mock_create, mock_ensure_path + ): # default args - look back 7 days generate_draft_bibxml_files_task() self.assertTrue(mock_ensure_path.called) self.assertCountEqual( - mock_create.call_args_list, [mock.call(self.young_event.doc, self.young_event.rev)] + mock_create.call_args_list, + [mock.call(self.young_event.doc, self.young_event.rev)], ) mock_create.reset_mock() mock_ensure_path.reset_mock() @@ -223,7 +265,9 @@ def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_en @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value(self, mock_create, mock_ensure_path): + def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value( + self, mock_create, mock_ensure_path + ): with self.assertRaises(ValueError): generate_draft_bibxml_files_task(days=0) self.assertFalse(mock_create.called) diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py new file mode 100644 index 00000000000..e4427b88b53 --- /dev/null +++ b/ietf/utils/searchindex.py @@ -0,0 +1,155 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +"""Search indexing utilities""" + +import re +from math import floor + +import httpx # just for exceptions +import typesense +import typesense.exceptions +from django.conf import settings + +from ietf.doc.models import Document, StoredObject +from ietf.doc.storage_utils import retrieve_str +from ietf.utils.log import log + +# Error classes that might succeed just by retrying a failed attempt. +# Must be a tuple for use with isinstance() +RETRYABLE_ERROR_CLASSES = ( + httpx.ConnectError, + httpx.ConnectTimeout, + typesense.exceptions.Timeout, + typesense.exceptions.ServerError, + typesense.exceptions.ServiceUnavailable, +) + + +DEFAULT_SETTINGS = { + "TYPESENSE_API_URL": "", + "TYPESENSE_API_KEY": "", + "TYPESENSE_COLLECTION_NAME": "docs", + "TASK_RETRY_DELAY": 10, + "TASK_MAX_RETRIES": 12, +} + + +def get_settings(): + return DEFAULT_SETTINGS | getattr(settings, "SEARCHINDEX_CONFIG", {}) + + +def enabled(): + _settings = get_settings() + return _settings["TYPESENSE_API_URL"] != "" + + +def _sanitize_text(content): + """Sanitize content or abstract text for search""" + # REs (with approximate names) + RE_DOT_OR_BANG_SPACE = r"\. |! " # -> " " (space) + RE_COMMENT_OR_TOC_CRUD = r"<--|-->|--+|\+|\.\.+" # -> "" + RE_BRACKETED_REF = r"\[[a-zA-Z0-9 -]+\]" # -> "" + RE_DOTTED_NUMBERS = r"[0-9]+\.[0-9]+(\.[0-9]+)?" # -> "" + RE_MULTIPLE_WHITESPACE = r"\s+" # -> " " (space) + # Replacement values (for clarity of intent) + SPACE = " " + EMPTY = "" + # Sanitizing begins here, order is significant! + content = re.sub(RE_DOT_OR_BANG_SPACE, SPACE, content.strip()) + content = re.sub(RE_COMMENT_OR_TOC_CRUD, EMPTY, content) + content = re.sub(RE_BRACKETED_REF, EMPTY, content) + content = re.sub(RE_DOTTED_NUMBERS, EMPTY, content) + content = re.sub(RE_MULTIPLE_WHITESPACE, SPACE, content) + return content.strip() + + +def update_or_create_rfc_entry(rfc: Document): + assert rfc.type_id == "rfc" + assert rfc.rfc_number is not None + + keywords: list[str] = rfc.keywords # help type checking + + subseries = rfc.part_of() + if len(subseries) > 1: + log( + f"RFC {rfc.rfc_number} is in multiple subseries. " + f"Indexing as {subseries[0].name}" + ) + subseries = subseries[0] if len(subseries) > 0 else None + obsoleted_by = rfc.relations_that("obs") + updated_by = rfc.relations_that("updates") + + stored_txt = ( + StoredObject.objects.exclude_deleted() + .filter(store="rfc", doc_name=rfc.name, name__startswith="txt/") + .first() + ) + content = "" + if stored_txt is not None: + # Should be available in the blobdb, but be cautious... + try: + content = retrieve_str(kind=stored_txt.store, name=stored_txt.name) + except Exception as err: + log(f"Unable to retrieve {stored_txt} from storage: {err}") + + ts_id = f"doc-{rfc.pk}" + ts_document = { + "rfcNumber": rfc.rfc_number, + "rfc": str(rfc.rfc_number), + "filename": rfc.name, + "title": rfc.title, + "abstract": _sanitize_text(rfc.abstract), + "keywords": keywords, + "type": "rfc", + "state": [state.name for state in rfc.states.all()], + "status": {"slug": rfc.std_level.slug, "name": rfc.std_level.name}, + "date": floor(rfc.time.timestamp()), + "publicationDate": floor(rfc.pub_datetime().timestamp()), + "stream": {"slug": rfc.stream.slug, "name": rfc.stream.name}, + "authors": [ + {"name": rfc_author.titlepage_name, "affiliation": rfc_author.affiliation} + for rfc_author in rfc.rfcauthor_set.all() + ], + "flags": { + "hiddenDefault": False, + "obsoleted": len(obsoleted_by) > 0, + "updated": len(updated_by) > 0, + }, + "obsoletedBy": [str(doc.rfc_number) for doc in obsoleted_by], + "updatedBy": [str(doc.rfc_number) for doc in updated_by], + "ranking": rfc.rfc_number, + } + if subseries is not None: + ts_document["subseries"] = { + "acronym": subseries.type.slug, + "number": int(subseries.name[len(subseries.type.slug) :]), + "total": len(subseries.contains()), + } + if rfc.group is not None: + ts_document["group"] = { + "acronym": rfc.group.acronym, + "name": rfc.group.name, + "full": f"{rfc.group.acronym} - {rfc.group.name}", + } + if ( + rfc.group.parent is not None + and rfc.stream_id not in ["ise", "irtf", "iab"] # exclude editorial? + ): + ts_document["area"] = { + "acronym": rfc.group.parent.acronym, + "name": rfc.group.parent.name, + "full": f"{rfc.group.parent.acronym} - {rfc.group.parent.name}", + } + if rfc.ad is not None: + ts_document["adName"] = rfc.ad.name + if content != "": + ts_document["content"] = _sanitize_text(content) + _settings = get_settings() + client = typesense.Client( + { + "api_key": _settings["TYPESENSE_API_KEY"], + "nodes": [_settings["TYPESENSE_API_URL"]], + } + ) + client.collections[_settings["TYPESENSE_COLLECTION_NAME"]].documents.upsert( + {"id": ts_id} | ts_document + ) diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py new file mode 100644 index 00000000000..8740716c85f --- /dev/null +++ b/ietf/utils/tests_searchindex.py @@ -0,0 +1,128 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from unittest import mock + +from django.conf import settings +from django.test.utils import override_settings + +from . import searchindex +from .test_utils import TestCase +from ..blobdb.models import Blob +from ..doc.factories import ( + WgDraftFactory, + WgRfcFactory, + PublishedRfcDocEventFactory, + BcpFactory, + StdFactory, +) +from ..doc.models import Document +from ..doc.storage_utils import store_str +from ..person.factories import PersonFactory + + +class SearchindexTests(TestCase): + def test_enabled(self): + with override_settings(): + try: + del settings.SEARCHINDEX_CONFIG + except AttributeError: + pass + self.assertFalse(searchindex.enabled()) + with override_settings( + SEARCHINDEX_CONFIG={"TYPESENSE_API_KEY": "this-is-not-a-key"} + ): + self.assertFalse(searchindex.enabled()) + with override_settings( + SEARCHINDEX_CONFIG={"TYPESENSE_API_URL": "http://example.com"} + ): + self.assertTrue(searchindex.enabled()) + + def test_sanitize_text(self): + dirty_text = """ + + This is text. It + is <---- full of \tprobl.....ems! Fix it. + """ + sanitized = "This is text It is full of problems Fix it." + self.assertEqual(searchindex._sanitize_text(dirty_text), sanitized) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_update_or_create_rfc_entry(self, mock_ts_client_constructor): + not_rfc = WgDraftFactory() + assert isinstance(not_rfc, Document) + with self.assertRaises(AssertionError): + searchindex.update_or_create_rfc_entry(not_rfc) + self.assertFalse(mock_ts_client_constructor.called) + + invalid_rfc = WgRfcFactory(name="rfc1000000", rfc_number=None) + assert isinstance(invalid_rfc, Document) + with self.assertRaises(AssertionError): + searchindex.update_or_create_rfc_entry(invalid_rfc) + self.assertFalse(mock_ts_client_constructor.called) + + rfc = PublishedRfcDocEventFactory().doc + assert isinstance(rfc, Document) + searchindex.update_or_create_rfc_entry(rfc) + self.assertTrue(mock_ts_client_constructor.called) + # walk the tree down to the method we expected to be called... + mock_upsert = mock_ts_client_constructor.return_value.collections[ + "frogs" + ].documents.upsert # matches value in override_settings above + self.assertTrue(mock_upsert.called) + upserted_dict = mock_upsert.call_args[0][0] + # Check a few values, not exhaustive + self.assertEqual(upserted_dict["id"], f"doc-{rfc.pk}") + self.assertEqual(upserted_dict["rfcNumber"], rfc.rfc_number) + self.assertEqual( + upserted_dict["abstract"], searchindex._sanitize_text(rfc.abstract) + ) + self.assertNotIn("adName", upserted_dict) + self.assertNotIn("content", upserted_dict) # no blob + self.assertNotIn("subseries", upserted_dict) + + # repeat, this time with contents, an AD, and subseries docs + mock_upsert.reset_mock() + store_str( + kind="rfc", + name=f"txt/{rfc.name}.txt", + content="The contents of this RFC", + doc_name=rfc.name, + doc_rev=rfc.rev, # expected to be None + ) + rfc.ad = PersonFactory(name="Alfred D. Rector") + # Put it in two Subseries docs to be sure this does not break things + # (the typesense schema does not support this for real at the moment) + BcpFactory(contains=[rfc], name="bcp1234") + StdFactory(contains=[rfc], name="std1234") + searchindex.update_or_create_rfc_entry(rfc) + self.assertTrue(mock_upsert.called) + upserted_dict = mock_upsert.call_args[0][0] + # Check a few values, not exhaustive + self.assertEqual( + upserted_dict["content"], + searchindex._sanitize_text("The contents of this RFC"), + ) + self.assertEqual(upserted_dict["adName"], "Alfred D. Rector") + self.assertIn("subseries", upserted_dict) + ss_dict = upserted_dict["subseries"] + # We should get one of the two subseries docs, but neither is more correct + # than the other... + self.assertTrue( + any( + ss_dict == {"acronym": ss_type, "number": 1234, "total": 1} + for ss_type in ["bcp", "std"] + ) + ) + + # Finally, delete the contents blob and make sure things don't blow up + mock_upsert.reset_mock() + Blob.objects.get(bucket="rfc", name=f"txt/{rfc.name}.txt").delete() + searchindex.update_or_create_rfc_entry(rfc) + self.assertTrue(mock_upsert.called) + upserted_dict = mock_upsert.call_args[0][0] + self.assertNotIn("content", upserted_dict) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 56e395c5ace..8c0c66cdf26 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2024, All Rights Reserved +# Copyright The IETF Trust 2007-2026, All Rights Reserved # -*- coding: utf-8 -*- from base64 import b64decode @@ -443,12 +443,8 @@ def _multiline_to_list(s): ), } RFCINDEX_DELETE_THEN_WRITE = False # S3Storage allows file_overwrite by default -RFCINDEX_OUTPUT_PATH = os.environ.get( - "DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/" -) -RFCINDEX_INPUT_PATH = os.environ.get( - "DATATRACKER_RFCINDEX_INPUT_PATH", "" -) +RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/") +RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH", "") # Configure the blobdb app for artifact storage _blobdb_replication_enabled = ( @@ -471,3 +467,13 @@ def _multiline_to_list(s): PASSWORD_POLICY_ENFORCE_AT_LOGIN = ( os.environ.get("DATATRACKER_ENFORCE_PW_POLICY", "true").lower() != "false" ) + +# Typesense search indexing +SEARCHINDEX_CONFIG = { + "TYPESENSE_API_URL": os.environ.get("DATATRACKER_TYPESENSE_API_URL", ""), + "TYPESENSE_API_KEY": os.environ.get("DATATRACKER_TYPESENSE_API_KEY", ""), + "TASK_RETRY_DELAY": os.environ.get("DATATRACKER_SEARCHINDEX_TASK_RETRY_DELAY", 10), + "TASK_MAX_RETRIES": os.environ.get( + "DATATRACKER_SEARCHINDEX_TASK_MAX_RETRIES", "12" + ), +} diff --git a/requirements.txt b/requirements.txt index 3d54b104ee6..2b8185dab96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -75,6 +75,7 @@ pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=2.0.0 # from TastyPie pytz==2025.2 # Pinned as changes need to be vetted for their effect on Meeting fields types-pytz==2025.2.0.20251108 # match pytz version +typesense>=2.0.0 requests>=2.32.4 types-requests>=2.32.4 requests-mock>=1.12.1 From e6a3b3ebc03ef539454cfa154ad0242b32c6a335 Mon Sep 17 00:00:00 2001 From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com> Date: Mon, 23 Mar 2026 15:46:07 +0000 Subject: [PATCH 040/188] ci: update base image target version to 20260323T1533 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index ce1828052ef..af43e990e00 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260304T1633 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260323T1533 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 6be54fb6b07..09f74cce287 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260304T1633 +20260323T1533 From 33f0dbf9e969a233f46251909515b249330fbb79 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 23 Mar 2026 12:39:15 -0500 Subject: [PATCH 041/188] feat: trigger red recomputation on RFC publication or metadata update (#10567) * feat: trigger red recomputation on RFC publication or metadata update * fix: move red precomputer call out of transaction * chore: remove old comment, simplify request call * fix: isolate delayed task in test * test: give settings_test an InMemoryStorage for r2-rfc * fix: follow obs/updates both ways when notifying red of changes * fix: improve red utils, test red and r2 utils * chore: ruff * chore: remove unused import * test: fix patch paths --------- Co-authored-by: Jennifer Richards --- ietf/api/serializers_rpc.py | 15 +++- ietf/api/tests_serializers_rpc.py | 88 ++++++++++++++++--- ietf/api/tests_views_rpc.py | 24 ++++- ietf/api/views_rpc.py | 12 ++- ietf/doc/tasks.py | 17 ++++ ietf/doc/tests_utils.py | 140 +++++++++++++++++++++++++++++- ietf/doc/utils_r2.py | 17 ++++ ietf/doc/utils_red.py | 31 +++++++ ietf/settings_test.py | 6 +- k8s/settings_local.py | 10 +++ 10 files changed, 341 insertions(+), 19 deletions(-) create mode 100644 ietf/doc/utils_r2.py create mode 100644 ietf/doc/utils_red.py diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 701f05eece8..397ca05d9b0 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -20,7 +20,7 @@ RfcAuthor, ) from ietf.doc.serializers import RfcAuthorSerializer -from ietf.doc.tasks import update_rfc_searchindex_task +from ietf.doc.tasks import trigger_red_precomputer_task, update_rfc_searchindex_task from ietf.doc.utils import ( default_consensus, prettify_std_name, @@ -683,7 +683,18 @@ def update(self, instance, validated_data): stale_subseries_relations.delete() if len(rfc_events) > 0: rfc.save_with_history(rfc_events) - + # Gather obs and updates in both directions as a title/author change to + # this doc affects the info rendering of all of the other RFCs + needs_updating = sorted( + [ + d.rfc_number + for d in [rfc] + + rfc.related_that_doc(("obs", "updates")) + + rfc.related_that(("obs", "updates")) + ] + ) + trigger_red_precomputer_task.delay(rfc_number_list=needs_updating) + # Update the search index also update_rfc_searchindex_task.delay(rfc.rfc_number) return rfc diff --git a/ietf/api/tests_serializers_rpc.py b/ietf/api/tests_serializers_rpc.py index ed326be451d..167ffcd3eec 100644 --- a/ietf/api/tests_serializers_rpc.py +++ b/ietf/api/tests_serializers_rpc.py @@ -1,4 +1,5 @@ # Copyright The IETF Trust 2026, All Rights Reserved + from unittest import mock from django.utils import timezone @@ -35,8 +36,21 @@ def test_create(self): serializer.save() @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") - def test_update(self, mock_update_searchindex_task): + @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task") + def test_update(self, mock_trigger_red_task, mock_update_searchindex_task): + updates = WgRfcFactory.create_batch(2) + obsoletes = WgRfcFactory.create_batch(2) rfc = WgRfcFactory(pages=10) + updated_by = WgRfcFactory.create_batch(2) + obsoleted_by = WgRfcFactory.create_batch(2) + for d in updates: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in obsoletes: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in updated_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + for d in obsoleted_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) serializer = EditableRfcSerializer( instance=rfc, data={ @@ -59,11 +73,6 @@ def test_update(self, mock_update_searchindex_task): ) self.assertTrue(serializer.is_valid()) result = serializer.save() - self.assertTrue(mock_update_searchindex_task.delay.called) - self.assertEqual( - mock_update_searchindex_task.delay.call_args, - mock.call(rfc.rfc_number), - ) result.refresh_from_db() self.assertEqual(result.title, "Yadda yadda yadda") self.assertEqual( @@ -91,12 +100,42 @@ def test_update(self, mock_update_searchindex_task): result.part_of(), [Document.objects.get(name="fyi999")], ) + # Confirm that red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_numbers = sorted( + [ + d.rfc_number + for d in [rfc] + updates + obsoletes + updated_by + obsoleted_by + ] + ) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was triggered correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") - def test_partial_update(self, mock_update_searchindex_task): + @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task") + def test_partial_update(self, mock_trigger_red_task, mock_update_searchindex_task): # We could test other permutations of fields, but authors is a partial update # we know we are going to use, so verifying that one in particular. + updates = WgRfcFactory.create_batch(2) + obsoletes = WgRfcFactory.create_batch(2) rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan") + updated_by = WgRfcFactory.create_batch(2) + obsoleted_by = WgRfcFactory.create_batch(2) + for d in updates: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in obsoletes: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in updated_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + for d in obsoleted_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) serializer = EditableRfcSerializer( partial=True, instance=rfc, @@ -113,11 +152,6 @@ def test_partial_update(self, mock_update_searchindex_task): ) self.assertTrue(serializer.is_valid()) result = serializer.save() - self.assertTrue(mock_update_searchindex_task.delay.called) - self.assertEqual( - mock_update_searchindex_task.delay.call_args, - mock.call(rfc.rfc_number), - ) result.refresh_from_db() self.assertEqual(rfc.title, "padawan") self.assertEqual( @@ -140,8 +174,27 @@ def test_partial_update(self, mock_update_searchindex_task): self.assertEqual(result.pages, 10) self.assertEqual(result.std_level_id, "ps") self.assertEqual(result.part_of(), []) + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_numbers = sorted( + [ + d.rfc_number + for d in [rfc] + updates + obsoletes + updated_by + obsoleted_by + ] + ) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) # Test only a field on the Document itself to be sure that it works + mock_trigger_red_task.delay.reset_mock() + mock_update_searchindex_task.delay.reset_mock() serializer = EditableRfcSerializer( partial=True, instance=rfc, @@ -151,3 +204,14 @@ def test_partial_update(self, mock_update_searchindex_task): result = serializer.save() result.refresh_from_db() self.assertEqual(rfc.title, "jedi master") + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index a679e747891..6d10bee8e8d 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -197,7 +197,8 @@ def test_notify_rfc_published(self, mock_task_delay): @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task") - def test_upload_rfc_files(self, mock_update_searchindex_task): + @mock.patch("ietf.api.views_rpc.trigger_red_precomputer_task") + def test_upload_rfc_files(self, mock_trigger_red_task, mock_update_searchindex_task): def _valid_post_data(): """Generate a valid post data dict @@ -218,7 +219,14 @@ def _valid_post_data(): } url = urlreverse("ietf.api.purple_api.upload_rfc_files") + updates = RfcFactory.create_batch(2) + obsoletes = RfcFactory.create_batch(2) + rfc = WgRfcFactory() + for r in obsoletes: + rfc.relateddocument_set.create(relationship_id="obs", target=r) + for r in updates: + rfc.relateddocument_set.create(relationship_id="updates", target=r) assert isinstance(rfc, Document), "WgRfcFactory should generate a Document" with TemporaryDirectory() as rfc_dir: settings.RFC_PATH = rfc_dir # affects overridden settings @@ -303,6 +311,7 @@ def _valid_post_data(): blob_in_the_way.delete() # valid post + mock_trigger_red_task.delay.reset_mock() r = self.client.post( url, _valid_post_data(), @@ -310,7 +319,6 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 200) - self.assertTrue(mock_update_searchindex_task.delay.called) self.assertEqual( mock_update_searchindex_task.delay.call_args, mock.call(rfc.rfc_number), @@ -350,6 +358,18 @@ def _valid_post_data(): b"This is .notprepped.xml", ".notprepped.xml blob should contain the expected content", ) + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_rfc_number_list = [rfc.rfc_number] + expected_rfc_number_list.extend( + [d.rfc_number for d in updates + obsoletes] + ) + expected_rfc_number_list = sorted(set(expected_rfc_number_list)) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) # re-post with replace = False should now fail mock_update_searchindex_task.reset_mock() diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index cb6a59a1674..59eed1e10e3 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -38,7 +38,11 @@ from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent from ietf.doc.serializers import RfcAuthorSerializer from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage -from ietf.doc.tasks import signal_update_rfc_metadata_task, update_rfc_searchindex_task +from ietf.doc.tasks import ( + signal_update_rfc_metadata_task, + trigger_red_precomputer_task, + update_rfc_searchindex_task, +) from ietf.person.models import Email, Person from ietf.sync.tasks import create_rfc_index_task @@ -516,6 +520,12 @@ def post(self, request): destination.parent.mkdir() shutil.move(ftm, destination) + # Trigger red precomputer + needs_updating = [rfc.rfc_number] + for rel in rfc.relateddocument_set.filter(relationship_id__in=["obs","updates"]): + needs_updating.append(rel.target.rfc_number) + trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating)) + # Trigger search index update update_rfc_searchindex_task.delay(rfc.rfc_number) return Response(NotificationAckSerializer().data) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index a38cd5eb5cf..19edb390149 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -7,11 +7,14 @@ import debug # pyflakes:ignore from celery import shared_task +from celery.exceptions import MaxRetriesExceededError from pathlib import Path from django.conf import settings from django.utils import timezone +from ietf.doc.utils_r2 import rfcs_are_in_r2 +from ietf.doc.utils_red import trigger_red_precomputer from ietf.utils import log, searchindex from ietf.utils.timezone import datetime_today @@ -169,6 +172,20 @@ def signal_update_rfc_metadata_task(rfc_number_list=()): signal_update_rfc_metadata(rfc_number_list) +@shared_task(bind=True) +def trigger_red_precomputer_task(self, rfc_number_list=()): + if not rfcs_are_in_r2(rfc_number_list): + log.log(f"Objects are not yet in R2 for RFCs {rfc_number_list}") + try: + countdown = getattr(settings, "RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", 10) + max_retries = getattr(settings, "RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", 12) + self.retry(countdown=countdown, max_retries=max_retries) + except MaxRetriesExceededError: + log.log(f"Gave up waiting for objects in R2 for RFCs {rfc_number_list}") + else: + trigger_red_precomputer(rfc_number_list) + + @shared_task(bind=True) def update_rfc_searchindex_task(self, rfc_number: int): """Update the search index for one RFC""" diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py index a2784bc85e0..ba672cd8479 100644 --- a/ietf/doc/tests_utils.py +++ b/ietf/doc/tests_utils.py @@ -1,15 +1,23 @@ # Copyright The IETF Trust 2020, All Rights Reserved import datetime +from io import BytesIO + +import mock import debug # pyflakes:ignore +import requests from pathlib import Path from unittest.mock import call, patch from django.conf import settings +from django.core.files.storage import storages from django.db import IntegrityError from django.test.utils import override_settings from django.utils import timezone + +from ietf.doc.utils_r2 import rfcs_are_in_r2 +from ietf.doc.utils_red import trigger_red_precomputer from ietf.group.factories import GroupFactory, RoleFactory from ietf.name.models import DocTagName from ietf.person.factories import PersonFactory @@ -17,11 +25,12 @@ from ietf.utils.test_utils import TestCase, name_of_file_containing, reload_db_objects from ietf.person.models import Person from ietf.doc.factories import DocumentFactory, WgRfcFactory, WgDraftFactory -from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor +from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor, StoredObject from ietf.doc.utils import (update_action_holders, add_state_change_event, update_documentauthors, fuzzy_find_documents, rebuild_reference_relations, build_file_urls, ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file, last_ballot_doc_revision) +from ietf.doc.storage_utils import store_str from ietf.utils.draft import Draft, PlaintextDraft from ietf.utils.xmldraft import XMLDraft @@ -559,3 +568,132 @@ def test_last_ballot_doc_revision(self): nobody = PersonFactory() self.assertIsNone(last_ballot_doc_revision(doc, nobody)) self.assertEqual(rev, last_ballot_doc_revision(doc, ad)) + + +class UtilsRedTests(TestCase): + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post") + def test_trigger_red_precomputer_not_configured(self, mock_post, mock_log): + with override_settings(): + try: + del settings.CUSTOM_SETTING_NAME + except AttributeError: + pass + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertEqual(mock_log.call_count, 1) + mock_args, _ = mock_log.call_args + self.assertEqual( + mock_args, + ("No URL configured for triggering red precompute multiple, skipping",), + ) + + mock_log.reset_mock() + with override_settings(TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL=None): + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertFalse(mock_post.called) + self.assertEqual(mock_log.call_count, 1) + mock_args, _ = mock_log.call_args + self.assertEqual( + mock_args, + ("No URL configured for triggering red precompute multiple, skipping",), + ) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + ) + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post", side_effect=requests.Timeout()) + def test_trigger_red_precomputer_swallows_timeout_exception( + self, mock_post, mock_log + ): + exception_raised = False + try: + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + except Exception: + exception_raised = True + self.assertFalse(exception_raised) + self.assertEqual(mock_log.call_count, 2) + # only checking the last log call + mock_args, _ = mock_log.call_args + self.assertEqual(len(mock_args), 1) + self.assertIn("POST request timed out", mock_args[0]) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + ) + @mock.patch("ietf.doc.utils_red.requests.post", side_effect=Exception()) + def test_trigger_red_precomputer_does_not_swallow_too_much(self, mock_post): + exception_raised = False + try: + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + except Exception: + exception_raised = True + self.assertTrue(exception_raised) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + DEFAULT_REQUESTS_TIMEOUT=314159265, + ) + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post") + def test_trigger_red_precomputer(self, mock_post, mock_log): + mock_post.return_value = mock.Mock(status_code=200) + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertTrue(mock_post.called) + _, mock_kwargs = mock_post.call_args + self.assertIn("url", mock_kwargs) + self.assertEqual(mock_kwargs["url"], "urlbits") + self.assertIn("json", mock_kwargs) + self.assertEqual(mock_kwargs["json"], {"rfcs": "1,2,3"}) + self.assertIn("timeout", mock_kwargs) + self.assertEqual(mock_kwargs["timeout"], 314159265) + self.assertEqual(mock_log.call_count, 1) # Not testing the first info log value + mock_log.reset_mock() + mock_post.reset_mock() + mock_post.return_value = mock.Mock( + status_code=500, + ) + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertEqual(mock_log.call_count, 2) + mock_args, _ = mock_log.call_args + self.assertEqual(len(mock_args), 1) + expected = f"POST request failed for {settings.TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL} : status_code=500" + self.assertEqual(mock_args[0], expected) + + +class UtilsR2TestCase(TestCase): + def test_rfcs_are_in_r2(self): + rfcs = WgRfcFactory.create_batch(2) + rfc_name_list = [rfc.name for rfc in rfcs] + rfc_number_list = [rfc.rfc_number for rfc in rfcs] + r2_rfc_bucket = storages["r2-rfc"] + # Right now the various doc Factories do not populate any content + self.assertEqual( + StoredObject.objects.filter( + store="rfc", doc_name__in=rfc_name_list + ).count(), + 0, + ) + self.assertTrue(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + for rfc in rfcs: + store_str( + kind="rfc", + name=f"testartifact/{rfc.name}.testartifact", + content="", + doc_name=rfc.name, + doc_rev=None, + ) + self.assertEqual( + StoredObject.objects.filter( + store="rfc", doc_name__in=rfc_name_list + ).count(), + 2, + ) + self.assertFalse(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + r2_rfc_bucket.save(f"testartifact/{rfcs[0].name}.testartifact", BytesIO(b"")) + self.assertFalse(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + r2_rfc_bucket.save(f"testartifact/{rfcs[1].name}.testartifact", BytesIO(b"")) + self.assertTrue(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + + + diff --git a/ietf/doc/utils_r2.py b/ietf/doc/utils_r2.py new file mode 100644 index 00000000000..53fb978303d --- /dev/null +++ b/ietf/doc/utils_r2.py @@ -0,0 +1,17 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.core.files.storage import storages + +from ietf.doc.models import StoredObject + + +def rfcs_are_in_r2(rfc_number_list=()): + r2_rfc_bucket = storages["r2-rfc"] + for rfc_number in rfc_number_list: + stored_objects = StoredObject.objects.filter( + store="rfc", doc_name=f"rfc{rfc_number}" + ) + for stored_object in stored_objects: + if not r2_rfc_bucket.exists(stored_object.name): + return False + return True diff --git a/ietf/doc/utils_red.py b/ietf/doc/utils_red.py new file mode 100644 index 00000000000..bcda893dca9 --- /dev/null +++ b/ietf/doc/utils_red.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import requests + +from django.conf import settings + +from ietf.utils.log import log + + +def trigger_red_precomputer(rfc_number_list=()): + url = getattr(settings, "TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None) + if url is not None: + payload = { + "rfcs": ",".join([str(n) for n in rfc_number_list]), + } + try: + log(f"Triggering red precompute multiple for RFCs {rfc_number_list}") + response = requests.post( + url=url, + json=payload, + timeout=settings.DEFAULT_REQUESTS_TIMEOUT, + ) + except requests.Timeout as e: + log(f"POST request timed out for {url} : {e}") + return + if response.status_code != 200: + log( + f"POST request failed for {url} : status_code={response.status_code}" + ) + else: + log("No URL configured for triggering red precompute multiple, skipping") diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 1f5a7e8ddca..e7ebc13eb28 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -115,8 +115,12 @@ def tempdir_with_cleanup(**kwargs): except NameError: pass -# Use InMemoryStorage for red bucket storage +# Use InMemoryStorage for red bucket and r2-rfc storages STORAGES["red_bucket"] = { "BACKEND": "django.core.files.storage.InMemoryStorage", "OPTIONS": {"location": "red_bucket"}, } +STORAGES["r2-rfc"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "r2-rfc"}, +} diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 8c0c66cdf26..323b7fd45a8 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -80,6 +80,16 @@ def _multiline_to_list(s): else: raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") +_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", None) +if _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY is not None: + RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY +_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", None) +if _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES is not None: + RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES +_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = os.environ.get("DATATRACKER_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None) +if _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL is not None: + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL + # Set DEBUG if DATATRACKER_DEBUG env var is the word "true" DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" From e5b037ba83c2275efcd5a034c4bd1af67932d23f Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 23 Mar 2026 13:19:54 -0500 Subject: [PATCH 042/188] fix: rebuild reference relations once we have rfc contents (#10578) Co-authored-by: Jennifer Richards --- ietf/api/tests_views_rpc.py | 13 ++++++++++++- ietf/api/views_rpc.py | 4 ++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 6d10bee8e8d..0db67e126f7 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -196,9 +196,15 @@ def test_notify_rfc_published(self, mock_task_delay): self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.rebuild_reference_relations_task") @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task") @mock.patch("ietf.api.views_rpc.trigger_red_precomputer_task") - def test_upload_rfc_files(self, mock_trigger_red_task, mock_update_searchindex_task): + def test_upload_rfc_files( + self, + mock_trigger_red_task, + mock_update_searchindex_task, + mock_rebuild_relations, + ): def _valid_post_data(): """Generate a valid post data dict @@ -370,6 +376,11 @@ def _valid_post_data(): self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) # Confirm that the search index update task was called correctly self.assertTrue(mock_update_searchindex_task.delay.called) + # Confirm reference relations rebuild task was called correctly + self.assertTrue(mock_rebuild_relations.delay.called) + _, mock_kwargs = mock_rebuild_relations.delay.call_args + self.assertIn("doc_names", mock_kwargs) + self.assertEqual(mock_kwargs["doc_names"], [rfc.name]) # re-post with replace = False should now fail mock_update_searchindex_task.reset_mock() diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 59eed1e10e3..6c7464e2525 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -40,6 +40,7 @@ from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage from ietf.doc.tasks import ( signal_update_rfc_metadata_task, + rebuild_reference_relations_task, trigger_red_precomputer_task, update_rfc_searchindex_task, ) @@ -527,6 +528,9 @@ def post(self, request): trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating)) # Trigger search index update update_rfc_searchindex_task.delay(rfc.rfc_number) + # Trigger reference relation srebuild + rebuild_reference_relations_task.delay(doc_names=[rfc.name]) + return Response(NotificationAckSerializer().data) From 10ebdf9a6433b34d32352b4bb1b4e9b285773de8 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 23 Mar 2026 15:44:22 -0300 Subject: [PATCH 043/188] chore: deduplicate logging and clean up config (#10592) * fix: remove redundant loggers + cleanup * style: ruff ruff (logging config) * chore: alphabetize loggers * refactor: modern suppression of DisallowedHost log * style: minor cleanup / comments * fix: roll back accidental commit * fix: django.request at ERROR level * fix: squelch other SuspiciousOperation mail --- ietf/settings.py | 188 +++++++++++++++++++---------------------------- 1 file changed, 74 insertions(+), 114 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index e0b4f201182..40a4cb5c56e 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2025, All Rights Reserved +# Copyright The IETF Trust 2007-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -13,6 +13,7 @@ import warnings from hashlib import sha384 from typing import Any, Dict, List, Tuple # pyflakes:ignore +from django.http import UnreadablePostError # DeprecationWarnings are suppressed by default, enable them warnings.simplefilter("always", DeprecationWarning) @@ -236,153 +237,112 @@ FILE_UPLOAD_PERMISSIONS = 0o644 -# ------------------------------------------------------------------------ -# Django/Python Logging Framework Modifications -# Filter out "Invalid HTTP_HOST" emails -# Based on http://www.tiwoc.de/blog/2013/03/django-prevent-email-notification-on-suspiciousoperation/ -from django.core.exceptions import SuspiciousOperation -def skip_suspicious_operations(record): - if record.exc_info: - exc_value = record.exc_info[1] - if isinstance(exc_value, SuspiciousOperation): - return False - return True +# +# Logging config +# -# Filter out UreadablePostError: -from django.http import UnreadablePostError +# Callback to filter out UnreadablePostError: def skip_unreadable_post(record): if record.exc_info: - exc_type, exc_value = record.exc_info[:2] # pylint: disable=unused-variable + exc_type, exc_value = record.exc_info[:2] # pylint: disable=unused-variable if isinstance(exc_value, UnreadablePostError): return False return True -# Copied from DEFAULT_LOGGING as of Django 1.10.5 on 22 Feb 2017, and modified -# to incorporate html logging, invalid http_host filtering, and more. -# Changes from the default has comments. - -# The Python logging flow is as follows: -# (see https://docs.python.org/2.7/howto/logging.html#logging-flow) -# -# Init: get a Logger: logger = logging.getLogger(name) -# -# Logging call, e.g. logger.error(level, msg, *args, exc_info=(...), extra={...}) -# --> Logger (discard if level too low for this logger) -# (create log record from level, msg, args, exc_info, extra) -# --> Filters (discard if any filter attach to logger rejects record) -# --> Handlers (discard if level too low for handler) -# --> Filters (discard if any filter attached to handler rejects record) -# --> Formatter (format log record and emit) -# - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - # - 'loggers': { - 'django': { - 'handlers': ['console', 'mail_admins'], - 'level': 'INFO', - }, - 'django.request': { - 'handlers': ['console'], - 'level': 'ERROR', + "version": 1, + "disable_existing_loggers": False, + "loggers": { + "celery": { + "handlers": ["console"], + "level": "INFO", }, - 'django.server': { - 'handlers': ['django.server'], - 'level': 'INFO', + "datatracker": { + "handlers": ["console"], + "level": "INFO", }, - 'django.security': { - 'handlers': ['console', ], - 'level': 'INFO', + "django": { + "handlers": ["console", "mail_admins"], + "level": "INFO", }, - 'oidc_provider': { - 'handlers': ['console', ], - 'level': 'DEBUG', + "django.request": {"level": "ERROR"}, # only log 5xx, ignore 4xx + "django.security": { + # SuspiciousOperation errors - log to console only + "handlers": ["console"], + "propagate": False, # no further handling please }, - 'datatracker': { - 'handlers': ['console'], - 'level': 'INFO', + "django.server": { + # Only used by Django's runserver development server + "handlers": ["django.server"], + "level": "INFO", }, - 'celery': { - 'handlers': ['console'], - 'level': 'INFO', + "oidc_provider": { + "handlers": ["console"], + "level": "DEBUG", }, }, - # - # No logger filters - # - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'plain', + "handlers": { + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "plain", }, - 'debug_console': { - # Active only when DEBUG=True - 'level': 'DEBUG', - 'filters': ['require_debug_true'], - 'class': 'logging.StreamHandler', - 'formatter': 'plain', + "debug_console": { + "level": "DEBUG", + "filters": ["require_debug_true"], + "class": "logging.StreamHandler", + "formatter": "plain", }, - 'django.server': { - 'level': 'INFO', - 'class': 'logging.StreamHandler', - 'formatter': 'django.server', + "django.server": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "django.server", }, - 'mail_admins': { - 'level': 'ERROR', - 'filters': [ - 'require_debug_false', - 'skip_suspicious_operations', # custom - 'skip_unreadable_posts', # custom + "mail_admins": { + "level": "ERROR", + "filters": [ + "require_debug_false", + "skip_unreadable_posts", ], - 'class': 'django.utils.log.AdminEmailHandler', - 'include_html': True, # non-default - } + "class": "django.utils.log.AdminEmailHandler", + "include_html": True, + }, }, - # # All these are used by handlers - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse', - }, - 'require_debug_true': { - '()': 'django.utils.log.RequireDebugTrue', + "filters": { + "require_debug_false": { + "()": "django.utils.log.RequireDebugFalse", }, - # custom filter, function defined above: - 'skip_suspicious_operations': { - '()': 'django.utils.log.CallbackFilter', - 'callback': skip_suspicious_operations, + "require_debug_true": { + "()": "django.utils.log.RequireDebugTrue", }, # custom filter, function defined above: - 'skip_unreadable_posts': { - '()': 'django.utils.log.CallbackFilter', - 'callback': skip_unreadable_post, + "skip_unreadable_posts": { + "()": "django.utils.log.CallbackFilter", + "callback": skip_unreadable_post, }, }, - # And finally the formatters - 'formatters': { - 'django.server': { - '()': 'django.utils.log.ServerFormatter', - 'format': '[%(server_time)s] %(message)s', + "formatters": { + "django.server": { + "()": "django.utils.log.ServerFormatter", + "format": "[%(server_time)s] %(message)s", }, - 'plain': { - 'style': '{', - 'format': '{levelname}: {name}:{lineno}: {message}', + "plain": { + "style": "{", + "format": "{levelname}: {name}:{lineno}: {message}", }, - 'json' : { + "json": { "class": "ietf.utils.jsonlogger.DatatrackerJsonFormatter", "style": "{", - "format": "{asctime}{levelname}{message}{name}{pathname}{lineno}{funcName}{process}", - } + "format": ( + "{asctime}{levelname}{message}{name}{pathname}{lineno}{funcName}" + "{process}{status_code}" + ), + }, }, } -# End logging -# ------------------------------------------------------------------------ - X_FRAME_OPTIONS = 'SAMEORIGIN' CSRF_TRUSTED_ORIGINS = [ From 14dd4cfdacb49552a2fcb9d9525e713f8ebd3c26 Mon Sep 17 00:00:00 2001 From: Tianyi Gao Date: Tue, 24 Mar 2026 02:45:24 +0800 Subject: [PATCH 044/188] feat: show parents on list of teams with grouping (#8635) (#10552) * feat: show parents on list of teams with grouping (#8635) * fix: sort teams by parent type then parent name in active teams list --- ietf/group/views.py | 13 +++++++-- ietf/templates/group/active_teams.html | 38 ++++++++++++++++---------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/ietf/group/views.py b/ietf/group/views.py index efe3eca15d5..8561a5059fc 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -245,10 +245,19 @@ def active_review_dirs(request): return render(request, 'group/active_review_dirs.html', {'dirs' : dirs }) def active_teams(request): - teams = Group.objects.filter(type="team", state="active").order_by("name") + parent_type_order = {"area": 1, "adm": 3, None: 4} + + def team_sort_key(group): + type_id = group.parent.type_id if group.parent else None + return (parent_type_order.get(type_id, 2), group.parent.name if group.parent else "", group.name) + + teams = sorted( + Group.objects.filter(type="team", state="active").select_related("parent"), + key=team_sort_key, + ) for group in teams: group.chairs = sorted(roles(group, "chair"), key=extract_last_name) - return render(request, 'group/active_teams.html', {'teams' : teams }) + return render(request, 'group/active_teams.html', {'teams': teams}) def active_iab(request): iabgroups = Group.objects.filter(type__in=("program","iabasg","iabworkshop"), state="active").order_by("-type_id","name") diff --git a/ietf/templates/group/active_teams.html b/ietf/templates/group/active_teams.html index 502d971a206..771dfda290a 100644 --- a/ietf/templates/group/active_teams.html +++ b/ietf/templates/group/active_teams.html @@ -16,21 +16,29 @@

Active teams

 - - {% for group in teams %} - - - - - - {% endfor %} - + {% regroup teams by parent as grouped_teams %} + {% for group_entry in grouped_teams %} + + + + + {% for group in group_entry.list %} + + + + + + {% endfor %} + + {% endfor %}
Parent + {{ group.parent.name }} + ({{ group.parent.acronym }}) +
Chairs
- {{ group.acronym }} - {{ group.name }} - {% for chair in group.chairs %} - {% person_link chair.person %}{% if not forloop.last %},{% endif %} - {% endfor %} -
+ {% if group_entry.grouper %}{{ group_entry.grouper.name }}{% else %}Other{% endif %} +
+ {{ group.acronym }} + {{ group.name }} + {% for chair in group.chairs %} + {% person_link chair.person %}{% if not forloop.last %},{% endif %} + {% endfor %} +
{% endblock %} {% block js %} From 057d52b76666ab6fcfd366700862be10db4844bb Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 23 Mar 2026 15:56:39 -0300 Subject: [PATCH 045/188] ci: update actions to avoid deprecations (#10604) * ci: upload-artifact -> v7 * ci: checkout -> v6 --- .github/workflows/build-base-app.yml | 2 +- .github/workflows/build-devblobstore.yml | 2 +- .github/workflows/build-mq-broker.yml | 2 +- .github/workflows/build.yml | 8 ++++---- .github/workflows/ci-run-tests.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/dev-assets-sync-nightly.yml | 2 +- .github/workflows/tests.yml | 14 +++++++------- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 4a4394fca0c..2b937cbfefc 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -18,7 +18,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: token: ${{ secrets.GH_COMMON_TOKEN }} diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml index f49a11af19c..41b2e0d47ab 100644 --- a/.github/workflows/build-devblobstore.yml +++ b/.github/workflows/build-devblobstore.yml @@ -20,7 +20,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index 4de861dbcd3..76c9b93168f 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -24,7 +24,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d97889fbb8e..8872c7f7d35 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,7 +66,7 @@ jobs: base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 fetch-tags: false @@ -164,7 +164,7 @@ jobs: TARGET_BASE: ${{needs.prepare.outputs.base_image_version}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 fetch-tags: false @@ -341,7 +341,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Upload Build Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: release-${{ env.PKG_VERSION }} path: /home/runner/work/release/release.tar.gz @@ -403,7 +403,7 @@ jobs: PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: ref: main diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml index 278bd8af2f3..5349f1ac7a6 100644 --- a/.github/workflows/ci-run-tests.yml +++ b/.github/workflows/ci-run-tests.yml @@ -23,7 +23,7 @@ jobs: base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 fetch-tags: false diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3444c03b5e0..4ab32d27a6a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Initialize CodeQL uses: github/codeql-action/init@v3 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 6d0683c471c..e255b270ff9 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: 'Dependency Review' uses: actions/dependency-review-action@v4 with: diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index 4cfbf6365b4..926d816b389 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -29,7 +29,7 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Login to GitHub Container Registry uses: docker/login-action@v3 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 836314bac09..be7b834b7af 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,7 +32,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-devblobstore:latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Prepare for tests run: | @@ -68,7 +68,7 @@ jobs: coverage xml - name: Upload geckodriver.log - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ failure() }} with: name: geckodriverlog @@ -87,7 +87,7 @@ jobs: mv latest-coverage.json coverage.json - name: Upload Coverage Results as Build Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} with: name: coverage @@ -102,7 +102,7 @@ jobs: project: [chromium, firefox] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: actions/setup-node@v6 with: @@ -121,7 +121,7 @@ jobs: npx playwright test --project=${{ matrix.project }} - name: Upload Report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} continue-on-error: true with: @@ -143,7 +143,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-db:latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Prepare for tests run: | @@ -180,7 +180,7 @@ jobs: npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js - name: Upload Report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} continue-on-error: true with: From 02070ee2f4dc6ee599e08a87e92b345198ae40fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 16:04:08 -0300 Subject: [PATCH 046/188] chore(deps): bump actions/setup-python from 5 to 6 (#9480) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8872c7f7d35..07a304cac22 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -175,7 +175,7 @@ jobs: node-version: 18.x - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.x" From 4a6627826993863bcb9cd5ede6b6e6f5b19eb0e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 17:07:28 -0300 Subject: [PATCH 047/188] chore(deps): bump ncipollo/release-action from 1.18.0 to 1.20.0 (#9478) Bumps [ncipollo/release-action](https://github.com/ncipollo/release-action) from 1.18.0 to 1.20.0. - [Release notes](https://github.com/ncipollo/release-action/releases) - [Commits](https://github.com/ncipollo/release-action/compare/v1.18.0...v1.20.0) --- updated-dependencies: - dependency-name: ncipollo/release-action dependency-version: 1.20.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 07a304cac22..ed425f9ae5d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -98,7 +98,7 @@ jobs: echo "IS_RELEASE=true" >> $GITHUB_ENV - name: Create Draft Release - uses: ncipollo/release-action@v1.18.0 + uses: ncipollo/release-action@v1.21.0 if: ${{ github.ref_name == 'release' }} with: prerelease: true @@ -315,7 +315,7 @@ jobs: histCoveragePath: historical-coverage.json - name: Create Release - uses: ncipollo/release-action@v1.18.0 + uses: ncipollo/release-action@v1.21.0 if: ${{ env.SHOULD_DEPLOY == 'true' }} with: allowUpdates: true @@ -328,7 +328,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Update Baseline Coverage - uses: ncipollo/release-action@v1.18.0 + uses: ncipollo/release-action@v1.21.0 if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }} with: allowUpdates: true From 753bd507c5d9cfdad4793d0e3feed68726fecf1e Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 23 Mar 2026 15:38:15 -0500 Subject: [PATCH 048/188] fix: include editorial docs in sent-to-rpc (#10605) --- ietf/api/views_rpc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 6c7464e2525..1e96118e581 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -210,19 +210,19 @@ def submitted_to_rpc(self, request): Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. """ ietf_docs = Q(states__type_id="draft-iesg", states__slug__in=["ann"]) - irtf_iab_ise_docs = Q( + irtf_iab_ise_editorial_docs = Q( states__type_id__in=[ "draft-stream-iab", "draft-stream-irtf", "draft-stream-ise", + "draft-stream-editorial", ], states__slug__in=["rfc-edit"], ) - # TODO: Need a way to talk about editorial stream docs docs = ( self.get_queryset() .filter(type_id="draft") - .filter(ietf_docs | irtf_iab_ise_docs) + .filter(ietf_docs | irtf_iab_ise_editorial_docs) ) serializer = self.get_serializer(docs, many=True) return Response(serializer.data) From 4308162174bb565b988c4fca9289c424c736ecba Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 23 Mar 2026 17:38:34 -0300 Subject: [PATCH 049/188] ci: handle rabbitmq version for push trigger (#10606) --- .github/workflows/build-mq-broker.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index 76c9b93168f..50472122c4c 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -39,6 +39,15 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Set rabbitmq version + id: rabbitmq-version + run: | + if [[ "${{ inputs.rabbitmq_version }}" == "" ]]; then + echo "RABBITMQ_VERSION=3.13-alpine" >> $GITHUB_OUTPUT + else + echo "RABBITMQ_VERSION=${{ inputs.rabbitmq_version }}" >> $GITHUB_OUTPUT + fi + - name: Docker Build & Push uses: docker/build-push-action@v6 env: @@ -48,7 +57,7 @@ jobs: file: dev/mq/Dockerfile platforms: linux/amd64,linux/arm64 push: true - build-args: RABBITMQ_VERSION=${{ inputs.rabbitmq_version }} + build-args: RABBITMQ_VERSION=${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }} tags: | - ghcr.io/ietf-tools/datatracker-mq:${{ inputs.rabbitmq_version }} + ghcr.io/ietf-tools/datatracker-mq:${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }} ghcr.io/ietf-tools/datatracker-mq:latest From eb041f7d81c7469f0b4c765150ccfadba5604177 Mon Sep 17 00:00:00 2001 From: Martin Thomson Date: Tue, 24 Mar 2026 05:39:01 +0900 Subject: [PATCH 050/188] fix: Rewrite CSS style attributes in SVG (#10584) This makes the dark mode work properly for drafts like https://datatracker.ietf.org/doc/html/draft-hajdusek-qirg-timing-physics-01 which have diagrams that use a mix of ordinary attributes and the style attribute. Using the style attribute makes the rules there invisible to the method we use for the remapping of black and white for dark mode. --- ietf/static/js/document_html.js | 79 +++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/ietf/static/js/document_html.js b/ietf/static/js/document_html.js index 6e8861739a6..3e609f3965d 100644 --- a/ietf/static/js/document_html.js +++ b/ietf/static/js/document_html.js @@ -117,4 +117,83 @@ document.addEventListener("DOMContentLoaded", function (event) { } }); } + + // Rewrite these CSS properties so that the values are available for restyling. + document.querySelectorAll("svg [style]").forEach(el => { + // Push these CSS properties into their own attributes + const SVG_PRESENTATION_ATTRS = new Set([ + 'alignment-baseline', 'baseline-shift', 'clip', 'clip-path', 'clip-rule', + 'color', 'color-interpolation', 'color-interpolation-filters', + 'color-rendering', 'cursor', 'direction', 'display', 'dominant-baseline', + 'fill', 'fill-opacity', 'fill-rule', 'filter', 'flood-color', + 'flood-opacity', 'font-family', 'font-size', 'font-size-adjust', + 'font-stretch', 'font-style', 'font-variant', 'font-weight', + 'image-rendering', 'letter-spacing', 'lighting-color', 'marker-end', + 'marker-mid', 'marker-start', 'mask', 'opacity', 'overflow', 'paint-order', + 'pointer-events', 'shape-rendering', 'stop-color', 'stop-opacity', + 'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', + 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', 'stroke-width', + 'text-anchor', 'text-decoration', 'text-rendering', 'unicode-bidi', + 'vector-effect', 'visibility', 'word-spacing', 'writing-mode', + ]); + + // Simple CSS splitter: respects quoted strings and parens so semicolons + // inside url(...) or "..." don't get treated as declaration boundaries. + function parseDeclarations(styleText) { + const decls = []; + let buf = ''; + let inStr = false; + let strChar = ''; + let escaped = false; + let depth = 0; + + for (const ch of styleText) { + if (inStr) { + if (escaped) { + escaped = false; + } else if (ch === '\\') { + escaped = true; + } else if (ch === strChar) { + inStr = false; + } + } else if (ch === '"' || ch === "'") { + inStr = true; + strChar = ch; + } else if (ch === '(') { + depth++; + } else if (ch === ')') { + depth--; + } else if (ch === ';' && depth === 0) { + const trimmed = buf.trim(); + if (trimmed) { + decls.push(trimmed); + } + buf = ''; + continue; + } + buf += ch; + } + const trimmed = buf.trim(); + if (trimmed) { + decls.push(trimmed); + } + return decls; + } + + const remainder = []; + for (const decl of parseDeclarations(el.getAttribute('style'))) { + const [prop, val] = decl.split(":", 2).map(v => v.trim()); + if (val && !/!important$/.test(val) && SVG_PRESENTATION_ATTRS.has(prop)) { + el.setAttribute(prop, val); + } else { + remainder.push(decl); + } + } + + if (remainder.length > 0) { + el.setAttribute('style', remainder.join('; ')); + } else { + el.removeAttribute('style'); + } + }); }); From 93e9bd3aad53808e791302c8fd99d74eb0873385 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 17:44:26 -0300 Subject: [PATCH 051/188] chore(deps): bump github/codeql-action from 3 to 4 (#9956) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3 to 4. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v3...v4) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4ab32d27a6a..bc20779ae6c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -29,9 +29,9 @@ jobs: uses: actions/checkout@v6 - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 From f9aebd5aa881557d7493db415d04a3d89494d637 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 17:48:56 -0300 Subject: [PATCH 052/188] chore(deps): bump actions/download-artifact from 4.3.0 to 6.0.0 (#9805) Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.3.0 to 6.0.0. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v4.3.0...v6.0.0) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: 6.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed425f9ae5d..74791747b63 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -186,7 +186,7 @@ jobs: - name: Download a Coverage Results if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v4.3.0 + uses: actions/download-artifact@v6.0.0 with: name: coverage @@ -291,7 +291,7 @@ jobs: - name: Download Coverage Results if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v4.3.0 + uses: actions/download-artifact@v6.0.0 with: name: coverage From 7d84aacad621b83753d6701afbcc864592d4822f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 17:50:26 -0300 Subject: [PATCH 053/188] chore(deps): bump the npm group (#10602) Bumps the npm group in /dev/deploy-to-container with 2 updates: [dockerode](https://github.com/apocas/dockerode) and [tar](https://github.com/isaacs/node-tar). Updates `dockerode` from 4.0.9 to 4.0.10 - [Release notes](https://github.com/apocas/dockerode/releases) - [Commits](https://github.com/apocas/dockerode/compare/v4.0.9...v4.0.10) Updates `tar` from 7.5.11 to 7.5.12 - [Release notes](https://github.com/isaacs/node-tar/releases) - [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md) - [Commits](https://github.com/isaacs/node-tar/compare/v7.5.11...v7.5.12) --- updated-dependencies: - dependency-name: dockerode dependency-version: 4.0.10 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: npm - dependency-name: tar dependency-version: 7.5.12 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: npm ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dev/deploy-to-container/package-lock.json | 113 ++++++++++------------ dev/deploy-to-container/package.json | 4 +- 2 files changed, 54 insertions(+), 63 deletions(-) diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json index b62109f0e21..a68f170c4b5 100644 --- a/dev/deploy-to-container/package-lock.json +++ b/dev/deploy-to-container/package-lock.json @@ -6,12 +6,12 @@ "": { "name": "deploy-to-container", "dependencies": { - "dockerode": "^4.0.9", + "dockerode": "^4.0.10", "fs-extra": "^11.3.4", "nanoid": "5.1.7", "nanoid-dictionary": "5.0.0", "slugify": "1.6.8", - "tar": "^7.5.11", + "tar": "^7.5.12", "yargs": "^17.7.2" }, "engines": { @@ -160,7 +160,6 @@ "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", - "license": "MIT", "dependencies": { "safer-buffer": "~2.1.0" } @@ -188,7 +187,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", - "license": "BSD-3-Clause", "dependencies": { "tweetnacl": "^0.14.3" } @@ -227,9 +225,9 @@ } }, "node_modules/buildcheck": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", - "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", "optional": true, "engines": { "node": ">=10.0.0" @@ -284,10 +282,9 @@ } }, "node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "license": "MIT", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "dependencies": { "ms": "^2.1.3" }, @@ -301,10 +298,9 @@ } }, "node_modules/docker-modem": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz", - "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", - "license": "Apache-2.0", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", "dependencies": { "debug": "^4.1.1", "readable-stream": "^3.5.0", @@ -316,14 +312,14 @@ } }, "node_modules/dockerode": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz", - "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==", + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.10.tgz", + "integrity": "sha512-8L/P9JynLBiG7/coiA4FlQXegHltRqS0a+KqI44P1zgQh8QLHTg7FKOwhkBgSJwZTeHsq30WRoVFLuwkfK0YFg==", "dependencies": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", - "docker-modem": "^5.0.6", + "docker-modem": "^5.0.7", "protobufjs": "^7.3.2", "tar-fs": "^2.1.4", "uuid": "^10.0.0" @@ -464,14 +460,12 @@ "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/nan": { - "version": "2.22.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", - "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", - "license": "MIT", + "version": "2.26.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz", + "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==", "optional": true }, "node_modules/nanoid": { @@ -580,8 +574,7 @@ "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/slugify": { "version": "1.6.8", @@ -594,13 +587,12 @@ "node_modules/split-ca": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", - "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", - "license": "ISC" + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" }, "node_modules/ssh2": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", - "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", "hasInstallScript": true, "dependencies": { "asn1": "^0.2.6", @@ -611,7 +603,7 @@ }, "optionalDependencies": { "cpu-features": "~0.0.10", - "nan": "^2.20.0" + "nan": "^2.23.0" } }, "node_modules/string_decoder": { @@ -647,9 +639,9 @@ } }, "node_modules/tar": { - "version": "7.5.11", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz", - "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==", + "version": "7.5.12", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.12.tgz", + "integrity": "sha512-9TsuLcdhOn4XztcQqhNyq1KOwOOED/3k58JAvtULiYqbO8B/0IBAAIE1hj0Svmm58k27TmcigyDI0deMlgG3uw==", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", @@ -698,8 +690,7 @@ "node_modules/tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", - "license": "Unlicense" + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, "node_modules/undici-types": { "version": "6.20.0", @@ -949,9 +940,9 @@ } }, "buildcheck": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", - "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", "optional": true }, "chownr": { @@ -993,17 +984,17 @@ } }, "debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "requires": { "ms": "^2.1.3" } }, "docker-modem": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz", - "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", "requires": { "debug": "^4.1.1", "readable-stream": "^3.5.0", @@ -1012,14 +1003,14 @@ } }, "dockerode": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz", - "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==", + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.10.tgz", + "integrity": "sha512-8L/P9JynLBiG7/coiA4FlQXegHltRqS0a+KqI44P1zgQh8QLHTg7FKOwhkBgSJwZTeHsq30WRoVFLuwkfK0YFg==", "requires": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", - "docker-modem": "^5.0.6", + "docker-modem": "^5.0.7", "protobufjs": "^7.3.2", "tar-fs": "^2.1.4", "uuid": "^10.0.0" @@ -1126,9 +1117,9 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "nan": { - "version": "2.22.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", - "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", + "version": "2.26.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz", + "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==", "optional": true }, "nanoid": { @@ -1213,14 +1204,14 @@ "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" }, "ssh2": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", - "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", "requires": { "asn1": "^0.2.6", "bcrypt-pbkdf": "^1.0.2", "cpu-features": "~0.0.10", - "nan": "^2.20.0" + "nan": "^2.23.0" } }, "string_decoder": { @@ -1250,9 +1241,9 @@ } }, "tar": { - "version": "7.5.11", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz", - "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==", + "version": "7.5.12", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.12.tgz", + "integrity": "sha512-9TsuLcdhOn4XztcQqhNyq1KOwOOED/3k58JAvtULiYqbO8B/0IBAAIE1hj0Svmm58k27TmcigyDI0deMlgG3uw==", "requires": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json index 1c95a4540ca..aa9e82dbdf8 100644 --- a/dev/deploy-to-container/package.json +++ b/dev/deploy-to-container/package.json @@ -2,12 +2,12 @@ "name": "deploy-to-container", "type": "module", "dependencies": { - "dockerode": "^4.0.9", + "dockerode": "^4.0.10", "fs-extra": "^11.3.4", "nanoid": "5.1.7", "nanoid-dictionary": "5.0.0", "slugify": "1.6.8", - "tar": "^7.5.11", + "tar": "^7.5.12", "yargs": "^17.7.2" }, "engines": { From 3d00e594e6a667fe8084bc8548d6993c29d515e6 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 23 Mar 2026 19:50:19 -0300 Subject: [PATCH 054/188] chore(deps): bump more action versions (#10608) --- .github/workflows/build-base-app.yml | 6 +++--- .github/workflows/build-devblobstore.yml | 6 +++--- .github/workflows/build-mq-broker.yml | 6 +++--- .github/workflows/build.yml | 10 +++++----- .github/workflows/dev-assets-sync-nightly.yml | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 2b937cbfefc..1b0855cc470 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -31,17 +31,17 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml index 41b2e0d47ab..14c4b1a135a 100644 --- a/.github/workflows/build-devblobstore.yml +++ b/.github/workflows/build-devblobstore.yml @@ -23,17 +23,17 @@ jobs: - uses: actions/checkout@v6 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index 50472122c4c..ef7ed2f65c0 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -30,10 +30,10 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} @@ -49,7 +49,7 @@ jobs: fi - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 74791747b63..8ec806b2291 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -253,10 +253,10 @@ jobs: EOL - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} @@ -267,7 +267,7 @@ jobs: run: echo "FEATURE_LATEST_TAG=$(echo $GITHUB_REF_NAME | tr / -)" >> $GITHUB_ENV - name: Build Images - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: @@ -360,7 +360,7 @@ jobs: steps: - name: Notify on Slack (Success) if: ${{ !contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v2 + uses: slackapi/slack-github-action@v3 with: token: ${{ secrets.SLACK_GH_BOT }} method: chat.postMessage @@ -375,7 +375,7 @@ jobs: value: "Completed" - name: Notify on Slack (Failure) if: ${{ contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v2 + uses: slackapi/slack-github-action@v3 with: token: ${{ secrets.SLACK_GH_BOT }} method: chat.postMessage diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index 926d816b389..cd986f06f33 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -32,14 +32,14 @@ jobs: - uses: actions/checkout@v6 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: From e51469a5d437491071610156d56dcb73191ad61c Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Fri, 27 Mar 2026 13:44:35 -0400 Subject: [PATCH 055/188] feat: add email/name for ADs and WG Chairs --- ietf/api/serializers_rpc.py | 28 +++++++++++++++++++++++++++- ietf/group/serializers.py | 6 ++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 397ca05d9b0..d888de4586c 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -27,7 +27,7 @@ update_action_holders, update_rfcauthors, ) -from ietf.group.models import Group +from ietf.group.models import Group, Role from ietf.group.serializers import AreaSerializer from ietf.name.models import StreamName, StdLevelName from ietf.person.models import Person @@ -97,6 +97,21 @@ class Meta: fields = ["draft_name", "authors"] +class WgChairSerializer(serializers.Serializer): + """Serialize a WG chair's name and email from a Role""" + + name = serializers.SerializerMethodField() + email = serializers.SerializerMethodField() + + @extend_schema_field(serializers.CharField) + def get_name(self, role: Role) -> str: + return role.person.plain_name() + + @extend_schema_field(serializers.EmailField) + def get_email(self, role: Role) -> str: + return role.email.email_address() + + class DocumentAuthorSerializer(serializers.ModelSerializer): """Serializer for a Person in a response""" @@ -126,6 +141,7 @@ class FullDraftSerializer(serializers.ModelSerializer): source="shepherd.person", read_only=True ) consensus = serializers.SerializerMethodField() + wg_chairs = serializers.SerializerMethodField() class Meta: model = Document @@ -145,11 +161,21 @@ class Meta: "consensus", "shepherd", "ad", + "wg_chairs", ] def get_consensus(self, doc: Document) -> Optional[bool]: return default_consensus(doc) + @extend_schema_field(WgChairSerializer(many=True)) + def get_wg_chairs(self, doc: Document): + if doc.group is None: + return [] + chairs = doc.group.role_set.filter(name_id="chair").select_related( + "person", "email" + ) + return WgChairSerializer(chairs, many=True).data + def get_source_format( self, doc: Document ) -> Literal["unknown", "xml-v2", "xml-v3", "txt"]: diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py index db3b37af48a..e789ba46bf7 100644 --- a/ietf/group/serializers.py +++ b/ietf/group/serializers.py @@ -20,8 +20,14 @@ class AreaDirectorSerializer(serializers.Serializer): Works with Email or Role """ + name = serializers.SerializerMethodField() email = serializers.SerializerMethodField() + @extend_schema_field(serializers.CharField) + def get_name(self, instance: Email | Role): + person = getattr(instance, 'person', None) + return person.plain_name() if person else None + @extend_schema_field(serializers.EmailField) def get_email(self, instance: Email | Role): if isinstance(instance, Role): From b1cc7edc7ff5e80f7eb0072657e88450a4b2c06b Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Fri, 27 Mar 2026 14:32:07 -0400 Subject: [PATCH 056/188] adapt test --- ietf/group/tests_serializers.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ietf/group/tests_serializers.py b/ietf/group/tests_serializers.py index bf29e6c8fd2..b584a17ae21 100644 --- a/ietf/group/tests_serializers.py +++ b/ietf/group/tests_serializers.py @@ -31,7 +31,7 @@ def test_serializes_role(self): serialized = AreaDirectorSerializer(role).data self.assertEqual( serialized, - {"email": role.email.email_address()}, + {"email": role.email.email_address(), "name": role.person.plain_name()}, ) def test_serializes_email(self): @@ -40,7 +40,10 @@ def test_serializes_email(self): serialized = AreaDirectorSerializer(email).data self.assertEqual( serialized, - {"email": email.email_address()}, + { + "email": email.email_address(), + "name": email.person.plain_name() if email.person else None, + }, ) @@ -63,7 +66,10 @@ def test_serializes_active_area(self): self.assertEqual(serialized["name"], area.name) self.assertCountEqual( serialized["ads"], - [{"email": ad.email.email_address()} for ad in ad_roles], + [ + {"email": ad.email.email_address(), "name": ad.person.plain_name()} + for ad in ad_roles + ], ) def test_serializes_inactive_area(self): From 5775077317640de8981cf27b0b8c54e42d8ae9a2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 1 Apr 2026 18:53:13 -0300 Subject: [PATCH 057/188] fix: limit access to manual post cancellation (#10638) * fix: drop access_token from URL * test: update test case * test: remove unneeded test There is no longer a dedicated manual post cancel action * chore: update copyrights --- ietf/submit/tests.py | 30 +++++++++++++++----------- ietf/templates/submit/manual_post.html | 16 ++++---------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 400d0d8c7d9..ad361d31b26 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2011-2023, All Rights Reserved +# Copyright The IETF Trust 2011-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -207,20 +207,24 @@ def test_manualpost_view(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertIn( - urlreverse( - "ietf.submit.views.submission_status", - kwargs=dict(submission_id=submission.pk) - ), - q("#manual.submissions td a").attr("href") - ) - self.assertIn( - submission.name, - q("#manual.submissions td a").text() + # Validate that the basic submission status URL is on the manual post page + # _without_ an access token, even if logged in as various users. + expected_url = urlreverse( + "ietf.submit.views.submission_status", + kwargs=dict(submission_id=submission.pk) ) + selected_elts = q("#manual.submissions td a") + self.assertEqual(expected_url, selected_elts.attr("href")) + self.assertIn(submission.name, selected_elts.text()) + for username in ["plain", "secretary"]: + self.client.login(username=username, password=username + "+password") + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + selected_elts = q("#manual.submissions td a") + self.assertEqual(expected_url, selected_elts.attr("href")) + self.assertIn(submission.name, selected_elts.text()) - def test_manualpost_cancel(self): - pass class SubmitTests(BaseSubmitTestCase): def setUp(self): diff --git a/ietf/templates/submit/manual_post.html b/ietf/templates/submit/manual_post.html index 6e4a2ba42ac..0da83e750fc 100644 --- a/ietf/templates/submit/manual_post.html +++ b/ietf/templates/submit/manual_post.html @@ -1,5 +1,5 @@ {% extends "submit/submit_base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin static %} {% block pagehead %} @@ -27,17 +27,9 @@

Submissions needing manual posting

{% for s in manual %} - {% if user.is_authenticated %} - - - {{ s.name }}-{{ s.rev }} - - - {% else %} - - {{ s.name }}-{{ s.rev }} - - {% endif %} + + {{ s.name }}-{{ s.rev }} + {{ s.submission_date }} {% if s.passes_checks %} From 6058769a64778679d4b3b5ca5e6937ed5f2ec6c8 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 2 Apr 2026 15:57:49 -0300 Subject: [PATCH 058/188] ci: optional bucket suffix for storage cfg (#10637) * ci: optional bucket suffix for storage cfg * style: ruff ruff * fix: roll back bizarre editor glitch --- docker/scripts/app-configure-blobstore.py | 10 +++++++--- k8s/settings_local.py | 22 ++++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py index 3140e393064..9ae64e00418 100755 --- a/docker/scripts/app-configure-blobstore.py +++ b/docker/scripts/app-configure-blobstore.py @@ -24,10 +24,13 @@ def init_blobstore(): ), ) for bucketname in ARTIFACT_STORAGE_NAMES: + adjusted_bucket_name = ( + os.environ.get("BLOB_STORE_BUCKET_PREFIX", "") + + bucketname + + os.environ.get("BLOB_STORE_BUCKET_SUFFIX", "") + ).strip() try: - blobstore.create_bucket( - Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip() - ) + blobstore.create_bucket(Bucket=adjusted_bucket_name) except botocore.exceptions.ClientError as err: if err.response["Error"]["Code"] == "BucketAlreadyExists": print(f"Bucket {bucketname} already exists") @@ -36,5 +39,6 @@ def init_blobstore(): else: print(f"Bucket {bucketname} created") + if __name__ == "__main__": sys.exit(init_blobstore()) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 323b7fd45a8..b45cbbe260c 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -18,7 +18,7 @@ def _multiline_to_list(s): - """Helper to split at newlines and conver to list""" + """Helper to split at newlines and convert to list""" return [item.strip() for item in s.split("\n")] @@ -80,13 +80,19 @@ def _multiline_to_list(s): else: raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") -_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", None) +_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = os.environ.get( + "DATATRACKER_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", None +) if _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY is not None: - RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY -_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", None) + RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY +_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = os.environ.get( + "DATATRACKER_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", None +) if _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES is not None: RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES -_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = os.environ.get("DATATRACKER_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None) +_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = os.environ.get( + "DATATRACKER_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None +) if _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL is not None: TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL @@ -387,6 +393,7 @@ def _multiline_to_list(s): "and DATATRACKER_BLOB_STORE_SECRET_KEY must be set" ) _blob_store_bucket_prefix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "") +_blob_store_bucket_suffix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_SUFFIX", "") _blob_store_enable_profiling = ( os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true" ) @@ -406,6 +413,9 @@ def _multiline_to_list(s): if storagename in ["staging"]: continue replica_storagename = f"r2-{storagename}" + adjusted_bucket_name = ( + _blob_store_bucket_prefix + storagename + _blob_store_bucket_suffix + ).strip() STORAGES[replica_storagename] = { "BACKEND": "ietf.doc.storage.MetadataS3Storage", "OPTIONS": dict( @@ -422,7 +432,7 @@ def _multiline_to_list(s): retries={"total_max_attempts": _blob_store_max_attempts}, ), verify=False, - bucket_name=f"{_blob_store_bucket_prefix}{storagename}".strip(), + bucket_name=adjusted_bucket_name, ietf_log_blob_timing=_blob_store_enable_profiling, ), } From a46a2efc05b2e7f5d1b50c76d543e1ca16ae8918 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Tue, 7 Apr 2026 09:25:24 +1200 Subject: [PATCH 059/188] feat: Generate bcp-index.txt (#10631) * feat: Generate bcp-index.txt * fix: Fix issue with author names * feat: Update bcp-index.txt header * refactor: Generalize some functions * fix: Sort RFCs * test: Add tests for bcp-index.txt * fix: Fix range bug * test: Add test for BCP entry * test: Fix test_create_bcp_txt_index --- ietf/sync/rfcindex.py | 98 +++++++++++++++++++++++++++++++ ietf/sync/tests_rfcindex.py | 69 ++++++++++++++++++++-- ietf/templates/sync/bcp-index.txt | 52 ++++++++++++++++ 3 files changed, 215 insertions(+), 4 deletions(-) create mode 100644 ietf/templates/sync/bcp-index.txt diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index 63c20449312..357cc4069a3 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -24,6 +24,8 @@ from ietf.utils.log import log FORMATS_FOR_INDEX = ["txt", "html", "pdf", "xml", "ps"] +SS_TXT_MARGIN = 3 +SS_TXT_CUE_COL_WIDTH = 14 def format_rfc_number(n): @@ -267,6 +269,87 @@ def get_rfc_text_index_entries(): return entries +def subseries_text_line(line, first=False): + """Return subseries text entry line""" + indent = " " * SS_TXT_CUE_COL_WIDTH + if first: + initial_indent = " " * SS_TXT_MARGIN + else: + initial_indent = indent + return fill( + line, + initial_indent=initial_indent, + subsequent_indent=indent, + width=80, + break_on_hyphens=False, + ) + + +def get_bcp_text_index_entries(): + """Returns BCP entries for bcp-index.txt""" + entries = [] + + highest_bcp_number = ( + Document.objects.filter(type_id="bcp") + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + .first() + .number + ) + + for bcp_number in range(1, highest_bcp_number + 1): + bcp_name = f"BCP{bcp_number}" + bcp = Document.objects.filter(type_id="bcp", name=f"{bcp_name.lower()}").first() + + if bcp: + entry = subseries_text_line( + ( + f"[{bcp_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(bcp_name) - 2 - SS_TXT_MARGIN)}" + f"Best Current Practice {bcp_number}," + ), + first=True, + ) + entry += "\n" + entry += subseries_text_line( + f"<{settings.RFC_EDITOR_INFO_BASE_URL}{bcp_name.lower()}>." + ) + entry += "\n" + entry += subseries_text_line( + "At the time of writing, this BCP comprises the following:" + ) + entry += "\n\n" + rfcs = sorted(bcp.contains(), key=lambda x: x.rfc_number) + for rfc in rfcs: + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + entry += subseries_text_line( + ( + f'{authors}, "{rfc.title}", BCP¶{bcp_number}, RFC¶{rfc.rfc_number}, ' + f"DOI¶{rfc.doi}, {rfc.pub_date().strftime('%B %Y')}, " + f"<{settings.RFC_EDITOR_INFO_BASE_URL}rfc{rfc.rfc_number}>." + ) + ).replace("¶", " ") + entry += "\n\n" + else: + entry = subseries_text_line( + ( + f"[{bcp_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(bcp_name) - 2 - SS_TXT_MARGIN)}" + f"Best Current Practice {bcp_number} currently contains no RFCs" + ), + first=True, + ) + entries.append(entry) + return entries + + def add_subseries_xml_index_entries(rfc_index, ss_type, include_all=False): """Add subseries entries for rfc-index.xml""" # subseries docs annotated with numeric number @@ -481,3 +564,18 @@ def create_rfc_xml_index(): pretty_print=4, ) save_to_red_bucket("rfc-index.xml", pretty_index) + + +def create_bcp_txt_index(): + """Create text index of BCPs""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating bcp-index.txt") + index = render_to_string( + "sync/bcp-index.txt", + { + "created_on": created_on, + "bcps": get_bcp_text_index_entries(), + }, + ) + save_to_red_bucket("bcp-index.txt", index) diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index e682c016f52..cad5b577d44 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -7,16 +7,22 @@ from django.test.utils import override_settings from lxml import etree -from ietf.doc.factories import PublishedRfcDocEventFactory, IndividualRfcFactory +from ietf.doc.factories import ( + BcpFactory, + IndividualRfcFactory, + PublishedRfcDocEventFactory, +) from ietf.name.models import DocTagName from ietf.sync.rfcindex import ( + create_bcp_txt_index, create_rfc_txt_index, create_rfc_xml_index, format_rfc_number, - save_to_red_bucket, - get_unusable_rfc_numbers, get_april1_rfc_numbers, get_publication_std_levels, + get_unusable_rfc_numbers, + save_to_red_bucket, + subseries_text_line, ) from ietf.utils.test_utils import TestCase @@ -69,6 +75,9 @@ def setUp(self): ).doc self.rfc.tags.add(DocTagName.objects.get(slug="errata")) + # Create a BCP with non-April Fools RFC + self.bcp = BcpFactory(contains=[self.rfc], name="bcp11") + # Set up a publication-std-levels.json file to indicate the publication # standard of self.rfc as different from its current value red_bucket.save( @@ -137,7 +146,7 @@ def test_create_rfc_xml_index(self, mock_save): children = list(index) # elements as list # Should be one rfc-not-issued-entry - self.assertEqual(len(children), 3) + self.assertEqual(len(children), 14) self.assertEqual( [ c.find(f"{ns}doc-id").text @@ -184,6 +193,53 @@ def test_create_rfc_xml_index(self, mock_save): [(f"{ns}month", "April"), (f"{ns}year", "2021")], ) + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_bcp_txt_index(self, mock_save): + create_bcp_txt_index() + self.assertEqual(mock_save.call_count, 1) + self.assertEqual(mock_save.call_args[0][0], "bcp-index.txt") + contents = mock_save.call_args[0][1] + self.assertTrue(isinstance(contents, str)) + # starts from 1 + self.assertIn( + "[BCP1]", + contents, + ) + # fill up to 11 + self.assertIn( + "[BCP10]", + contents, + ) + # but not to 12 + self.assertNotIn( + "[BCP12]", + contents, + ) + # Test empty BCPs + self.assertIn( + "Best Current Practice 9 currently contains no RFCs", + contents, + ) + # No zero prefix! + self.assertNotIn( + "[BCP0001]", + contents, + ) + # Has BCP11 with a RFC + self.assertIn( + "Best Current Practice 11,", + contents, + ) + self.assertIn( + f'"{self.rfc.title}"', + contents, + ) + self.assertIn( + f'BCP 11, RFC {self.rfc.rfc_number},', + contents, + ) + class HelperTests(TestCase): def test_format_rfc_number(self): @@ -234,3 +290,8 @@ def test_get_publication_std_levels_raises(self): with self.assertRaises(json.JSONDecodeError): get_publication_std_levels() red_bucket.delete("publication-std-levels.json") + + def test_subseries_text_line(self): + text = "foobar" + self.assertEqual(subseries_text_line(line=text, first=True), f" {text}") + self.assertEqual(subseries_text_line(line=text), f" {text}") diff --git a/ietf/templates/sync/bcp-index.txt b/ietf/templates/sync/bcp-index.txt new file mode 100644 index 00000000000..dd19920eba8 --- /dev/null +++ b/ietf/templates/sync/bcp-index.txt @@ -0,0 +1,52 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + BCP INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all BCPs in numeric order. The BCPs +form a sub-series of the RFC document series, specifically those RFCs +with the status BEST CURRENT PRACTICE. + +BCP citations appear in this format: + + [BCP#] Best Current Practice #, + . + At the time of writing, this BCP comprises the following: + + Author 1, Author 2, "Title of the RFC", BCP #, RFC №, + DOI DOI string, Issue date, + . + +For example: + + [BCP3] Best Current Practice 3, + . + At the time of writing, this BCP comprises the following: + + F. Kastenholz, "Variance for The PPP Compression Control Protocol + and The PPP Encryption Control Protocol", BCP 3, RFC 1915, + DOI 10.17487/RFC1915, February 1996, + . + +Key to fields: + +# is the BCP number. + +№ is the RFC number. + +BCPs and other RFCs may be obtained from https://www.rfc-editor.org. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + BCP INDEX + --------- + + + +{% for bcp in bcps %}{{bcp|safe}} + +{% endfor %} From 7c7219f0dcf326f369c7b4bd04337f95f0a7a9f4 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 8 Apr 2026 11:16:57 +1200 Subject: [PATCH 060/188] feat: Generate std-index.txt (#10665) * feat: Generate std-index.txt * style: Ruff ruff Good boy! * test: Fix flaky test * test: Add tests for std-index.txt --- ietf/sync/rfcindex.py | 80 +++++++++++++++++++++++++++++++ ietf/sync/tests_rfcindex.py | 64 ++++++++++++++++++++++++- ietf/templates/sync/std-index.txt | 51 ++++++++++++++++++++ 3 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 ietf/templates/sync/std-index.txt diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index 357cc4069a3..6a6a4bfa9f8 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -350,6 +350,71 @@ def get_bcp_text_index_entries(): return entries +def get_std_text_index_entries(): + """Returns STD entries for std-index.txt""" + entries = [] + + highest_std_number = ( + Document.objects.filter(type_id="std") + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + .first() + .number + ) + + for std_number in range(1, highest_std_number + 1): + std_name = f"STD{std_number}" + std = Document.objects.filter(type_id="std", name=f"{std_name.lower()}").first() + + if std and std.contains(): + entry = subseries_text_line( + ( + f"[{std_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(std_name) - 2 - SS_TXT_MARGIN)}" + f"Internet Standard {std_number}," + ), + first=True, + ) + entry += "\n" + entry += subseries_text_line( + f"<{settings.RFC_EDITOR_INFO_BASE_URL}{std_name.lower()}>." + ) + entry += "\n" + entry += subseries_text_line( + "At the time of writing, this STD comprises the following:" + ) + entry += "\n\n" + rfcs = sorted(std.contains(), key=lambda x: x.rfc_number) + for rfc in rfcs: + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + entry += subseries_text_line( + ( + f'{authors}, "{rfc.title}", STD¶{std_number}, RFC¶{rfc.rfc_number}, ' + f"DOI¶{rfc.doi}, {rfc.pub_date().strftime('%B %Y')}, " + f"<{settings.RFC_EDITOR_INFO_BASE_URL}rfc{rfc.rfc_number}>." + ) + ).replace("¶", " ") + entry += "\n\n" + else: + entry = subseries_text_line( + ( + f"[{std_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(std_name) - 2 - SS_TXT_MARGIN)}" + f"Internet Standard {std_number} currently contains no RFCs" + ), + first=True, + ) + entries.append(entry) + return entries + + def add_subseries_xml_index_entries(rfc_index, ss_type, include_all=False): """Add subseries entries for rfc-index.xml""" # subseries docs annotated with numeric number @@ -579,3 +644,18 @@ def create_bcp_txt_index(): }, ) save_to_red_bucket("bcp-index.txt", index) + + +def create_std_txt_index(): + """Create text index of STDs""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating std-index.txt") + index = render_to_string( + "sync/std-index.txt", + { + "created_on": created_on, + "stds": get_std_text_index_entries(), + }, + ) + save_to_red_bucket("std-index.txt", index) diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index cad5b577d44..70bc41b9921 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -9,6 +9,7 @@ from ietf.doc.factories import ( BcpFactory, + StdFactory, IndividualRfcFactory, PublishedRfcDocEventFactory, ) @@ -17,6 +18,7 @@ create_bcp_txt_index, create_rfc_txt_index, create_rfc_xml_index, + create_std_txt_index, format_rfc_number, get_april1_rfc_numbers, get_publication_std_levels, @@ -78,6 +80,9 @@ def setUp(self): # Create a BCP with non-April Fools RFC self.bcp = BcpFactory(contains=[self.rfc], name="bcp11") + # Create a STD with non-April Fools RFC + self.std = StdFactory(contains=[self.rfc], name="std11") + # Set up a publication-std-levels.json file to indicate the publication # standard of self.rfc as different from its current value red_bucket.save( @@ -146,7 +151,7 @@ def test_create_rfc_xml_index(self, mock_save): children = list(index) # elements as list # Should be one rfc-not-issued-entry - self.assertEqual(len(children), 14) + self.assertEqual(len(children), 15) self.assertEqual( [ c.find(f"{ns}doc-id").text @@ -236,7 +241,62 @@ def test_create_bcp_txt_index(self, mock_save): contents, ) self.assertIn( - f'BCP 11, RFC {self.rfc.rfc_number},', + "BCP 11,", + contents, + ) + self.assertIn( + f"RFC {self.rfc.rfc_number},", + contents, + ) + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_std_txt_index(self, mock_save): + create_std_txt_index() + self.assertEqual(mock_save.call_count, 1) + self.assertEqual(mock_save.call_args[0][0], "std-index.txt") + contents = mock_save.call_args[0][1] + self.assertTrue(isinstance(contents, str)) + # starts from 1 + self.assertIn( + "[STD1]", + contents, + ) + # fill up to 11 + self.assertIn( + "[STD10]", + contents, + ) + # but not to 12 + self.assertNotIn( + "[STD12]", + contents, + ) + # Test empty STDs + self.assertIn( + "Internet Standard 9 currently contains no RFCs", + contents, + ) + # No zero prefix! + self.assertNotIn( + "[STD0001]", + contents, + ) + # Has STD11 with a RFC + self.assertIn( + "Internet Standard 11,", + contents, + ) + self.assertIn( + f'"{self.rfc.title}"', + contents, + ) + self.assertIn( + "STD 11,", + contents, + ) + self.assertIn( + f"RFC {self.rfc.rfc_number},", contents, ) diff --git a/ietf/templates/sync/std-index.txt b/ietf/templates/sync/std-index.txt new file mode 100644 index 00000000000..c075d1d43e4 --- /dev/null +++ b/ietf/templates/sync/std-index.txt @@ -0,0 +1,51 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + STD INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all STDs in numeric order. Each +STD represents a single Internet Standard technical specification, +composed of one or more RFCs with Internet Standard status. + +STD citations appear in this format: + + [STD#] Best Current Practice #, + . + At the time of writing, this STD comprises the following: + + Author 1, Author 2, "Title of the RFC", STD #, RFC №, + DOI DOI string, Issue date, + . + +For example: + + [STD6] Internet Standard 6, + . + At the time of writing, this STD comprises the following: + + J. Postel, "User Datagram Protocol", STD 6, RFC 768, + DOI 10.17487/RFC0768, August 1980, + . + +Key to fields: + +# is the STD number. + +№ is the RFC number. + +STDs and other RFCs may be obtained from https://www.rfc-editor.org. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + STD INDEX + --------- + + + +{% for std in stds %}{{std|safe}} + +{% endfor %} From e72ead86dee707b5cbd9aeea96437dbaee78c88d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 12:35:06 -0300 Subject: [PATCH 061/188] chore(deps): bump appleboy/ssh-action from 1.2.2 to 1.2.5 (#10623) Bumps [appleboy/ssh-action](https://github.com/appleboy/ssh-action) from 1.2.2 to 1.2.5. - [Release notes](https://github.com/appleboy/ssh-action/releases) - [Commits](https://github.com/appleboy/ssh-action/compare/2ead5e36573f08b82fbfce1504f1a4b05a647c6f...0ff4204d59e8e51228ff73bce53f80d53301dee2) --- updated-dependencies: - dependency-name: appleboy/ssh-action dependency-version: 1.2.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/tests-az.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml index 8553563a192..833ca89befd 100644 --- a/.github/workflows/tests-az.yml +++ b/.github/workflows/tests-az.yml @@ -38,7 +38,7 @@ jobs: ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts - name: Remote SSH into VM - uses: appleboy/ssh-action@2ead5e36573f08b82fbfce1504f1a4b05a647c6f + uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: From c7657c3f22f5f7a906fd2cf01aaed7b54feca9e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 12:37:35 -0300 Subject: [PATCH 062/188] chore(deps): bump stefanzweifel/git-auto-commit-action from 6 to 7 (#10624) Bumps [stefanzweifel/git-auto-commit-action](https://github.com/stefanzweifel/git-auto-commit-action) from 6 to 7. - [Release notes](https://github.com/stefanzweifel/git-auto-commit-action/releases) - [Changelog](https://github.com/stefanzweifel/git-auto-commit-action/blob/master/CHANGELOG.md) - [Commits](https://github.com/stefanzweifel/git-auto-commit-action/compare/v6...v7) --- updated-dependencies: - dependency-name: stefanzweifel/git-auto-commit-action dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-base-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 1b0855cc470..5e274838a13 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -60,7 +60,7 @@ jobs: echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE - name: Commit CHANGELOG.md - uses: stefanzweifel/git-auto-commit-action@v6 + uses: stefanzweifel/git-auto-commit-action@v7 with: branch: ${{ github.ref_name }} commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}' From f39e916a73eaab6c0172a09e98c28ba628b7bcc4 Mon Sep 17 00:00:00 2001 From: Eric Rescorla Date: Wed, 8 Apr 2026 08:50:19 -0700 Subject: [PATCH 063/188] fix: Rewrite upper right document search box (#10538) * Rewrite upper right document search box. Fixes #10358 This is a fix to the problem where the first item in the dropdown is auto-selected and then when you hit return you go to that rather than searching for what's in the text field. It appears to be challenging to get this behavior with select2, so this is actually a rewrite of the box with explicit behavior. As a side effect, the draft names actually render a bit better. Co-Authored-By: Claude Opus 4.6 * Respond to review comments --------- Co-authored-by: EKR aibot Co-authored-by: Claude Opus 4.6 --- ietf/static/css/ietf.scss | 17 +++++ ietf/static/js/navbar-doc-search.js | 113 ++++++++++++++++++++++++++++ ietf/templates/base.html | 24 +++--- package.json | 1 + 4 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 ietf/static/js/navbar-doc-search.js diff --git a/ietf/static/css/ietf.scss b/ietf/static/css/ietf.scss index df973863d53..6695c57b13d 100644 --- a/ietf/static/css/ietf.scss +++ b/ietf/static/css/ietf.scss @@ -1216,3 +1216,20 @@ iframe.status { .overflow-shadows--bottom-only { box-shadow: inset 0px -21px 18px -20px var(--bs-body-color); } + +#navbar-doc-search-wrapper { + position: relative; +} + +#navbar-doc-search-results { + max-height: 400px; + overflow-y: auto; + min-width: auto; + left: 0; + right: 0; + + .dropdown-item { + white-space: normal; + overflow-wrap: break-word; + } +} diff --git a/ietf/static/js/navbar-doc-search.js b/ietf/static/js/navbar-doc-search.js new file mode 100644 index 00000000000..c36c0323103 --- /dev/null +++ b/ietf/static/js/navbar-doc-search.js @@ -0,0 +1,113 @@ +$(function () { + var $input = $('#navbar-doc-search'); + var $results = $('#navbar-doc-search-results'); + var ajaxUrl = $input.data('ajax-url'); + var debounceTimer = null; + var highlightedIndex = -1; + var keyboardHighlight = false; + var currentItems = []; + + function showDropdown() { + $results.addClass('show'); + } + + function hideDropdown() { + $results.removeClass('show'); + highlightedIndex = -1; + keyboardHighlight = false; + updateHighlight(); + } + + function updateHighlight() { + $results.find('.dropdown-item').removeClass('active'); + if (highlightedIndex >= 0 && highlightedIndex < currentItems.length) { + $results.find('.dropdown-item').eq(highlightedIndex).addClass('active'); + } + } + + function doSearch(query) { + if (query.length < 2) { + hideDropdown(); + return; + } + $.ajax({ + url: ajaxUrl, + dataType: 'json', + data: { q: query }, + success: function (data) { + currentItems = data; + highlightedIndex = -1; + $results.empty(); + if (data.length === 0) { + $results.append('
  • No results found
    • '); + } else { + data.forEach(function (item) { + var $li = $('
  • '); + var $a = $('' + item.text + ''); + $li.append($a); + $results.append($li); + }); + } + showDropdown(); + } + }); + } + + $input.on('input', function () { + clearTimeout(debounceTimer); + var query = $(this).val().trim(); + debounceTimer = setTimeout(function () { + doSearch(query); + }, 250); + }); + + $input.on('keydown', function (e) { + if (e.key === 'ArrowDown') { + e.preventDefault(); + if (highlightedIndex < currentItems.length - 1) { + highlightedIndex++; + keyboardHighlight = true; + updateHighlight(); + } + } else if (e.key === 'ArrowUp') { + e.preventDefault(); + if (highlightedIndex > 0) { + highlightedIndex--; + keyboardHighlight = true; + updateHighlight(); + } + } else if (e.key === 'Enter') { + e.preventDefault(); + if (keyboardHighlight && highlightedIndex >= 0 && highlightedIndex < currentItems.length) { + window.location.href = currentItems[highlightedIndex].url; + } else { + var query = $(this).val().trim(); + if (query) { + window.location.href = '/doc/search/?name=' + encodeURIComponent(query) + '&rfcs=on&activedrafts=on&olddrafts=on'; + } + } + } else if (e.key === 'Escape') { + hideDropdown(); + $input.blur(); + } + }); + + // Hover highlights (visual only — Enter still submits the text) + $results.on('mouseenter', '.dropdown-item', function () { + highlightedIndex = $results.find('.dropdown-item').index(this); + keyboardHighlight = false; + updateHighlight(); + }); + + $results.on('mouseleave', '.dropdown-item', function () { + highlightedIndex = -1; + updateHighlight(); + }); + + // Click outside closes dropdown + $(document).on('click', function (e) { + if (!$(e.target).closest('#navbar-doc-search-wrapper').length) { + hideDropdown(); + } + }); +}); diff --git a/ietf/templates/base.html b/ietf/templates/base.html index 25ce50c4672..b0df04f30a3 100644 --- a/ietf/templates/base.html +++ b/ietf/templates/base.html @@ -67,13 +67,17 @@ {% endif %} - +
    • -
  • - +
  • + Statistics -
  • diff --git a/ietf/templates/stats/annual_report_inputs.html b/ietf/templates/stats/annual_report_inputs.html new file mode 100644 index 00000000000..15add7ece3c --- /dev/null +++ b/ietf/templates/stats/annual_report_inputs.html @@ -0,0 +1,55 @@ +{# Copyright The IETF Trust 2026, All Rights Reserved #} +{% extends "base.html" %} +{% load origin %} +{% load ietf_filters static %} +{% block content %} + {% origin %} +

    {% block title %}Annual Report Inputs for {{ year }}{% endblock %}

    +
    +
    + + + +
    +
    +

    Summary

    + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Email addressesUnique persons foundAddresses with no person recordApproximate unique people
    Authors{{ author_count }}{{ author_person_count }}{{ author_noperson_count }}{{ author_person_count|add:author_noperson_count }}
    Submitters{{ submitter_count }}{{ submitter_person_count }}{{ submitter_noperson_count }}{{ submitter_person_count|add:submitter_noperson_count }}
    +

    Drafts submitted in {{ year }}: {{ draft_count }}

    +

    Downloads

    + +{% endblock %} diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py new file mode 100755 index 00000000000..9a969a52178 --- /dev/null +++ b/ietf/utils/reports.py @@ -0,0 +1,49 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved + +from typing import List, Set, Tuple +from django.db.models import QuerySet + +from email.utils import parseaddr + +from ietf.person.models import Person +from ietf.submit.models import Submission + + +def authors_by_year(year: int) -> Set[str]: + """Email addresses provided by I-D authors for drafts that were submitted in the given year.""" + addresses = set() + for submission in Submission.objects.filter( + submission_date__year=year, state="posted" + ): + addresses.update([a["email"] for a in submission.authors]) + return addresses + + +def submitters_by_year(year: int) -> Set[str]: + """Email addresses provided by I-D submitters for drafts that were submitted in the given year.""" + return set( + [ + parseaddr(a)[1] + for a in Submission.objects.filter( + submitter__contains="@", submission_date__year=year, state="posted" + ).values_list("submitter", flat=True) + ] + ) + + +def unique_people(addresses: List[str]) -> Tuple["QuerySet[Person]", Set]: + """Identify Person records matching email addresses and email addresses with no Person record. + + Given a list of email addresses, return + ( + a list of unique Person records with a matching email address, + a list of unique email addresses with no matching Person record + ) + The sum of the lengths of these lists is a best-approximation for how + many unique people the list of addresses belong to. + """ + persons = Person.objects.filter(email__address__in=addresses).distinct() + known_email = set(persons.values_list("email__address", flat=True)) + return (persons, set(addresses) - set(known_email)) + + diff --git a/ietf/utils/tests_reports.py b/ietf/utils/tests_reports.py new file mode 100755 index 00000000000..83daa15cc14 --- /dev/null +++ b/ietf/utils/tests_reports.py @@ -0,0 +1,189 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime + +import debug # pyflakes:ignore + +from ietf.doc.factories import IndividualDraftFactory +from ietf.person.factories import EmailFactory +from ietf.person.models import Person, Email +from ietf.submit.factories import SubmissionFactory +from ietf.utils.reports import authors_by_year, submitters_by_year, unique_people +from ietf.utils.test_utils import TestCase + + +class ReportTests(TestCase): + def setUp(self): + super().setUp() + + # Build 5 drafts submitted across two years, with 6 unique authors, + # one of which has more than one email address (author0@example.com and + # author0@example.net). The drafts are submitted by two of the six authors, + # again using multiple addresses for author0, and two identities that are not authors. + # Then build a draft where the submission's submitter info doesn't contain an email + # address (we have those in the production database) to make sure that submitter isn't + # counted. + + self.make_draft_submission( + year=2020, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 1", "author1@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=3, + day=3, + submitter_name="NotanAuthor 0", + submitter_email="notanauthor0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.com"), # Note alternate email + ("Author 3", "author3@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=12, + day=31, + submitter_name="Author 3", + submitter_email="author3@example.net", + author_nameaddrs=[("Author 3", "author3@example.net")], + ) + + self.make_draft_submission( + year=2021, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.com", # Note alternate email + author_nameaddrs=[ + ("Author 0", "author0@example.com"), # Again, alternate email + ("Author 4", "author4@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="NoatanAuthor 2", + submitter_email="notanauthor2@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 3", "author3@example.net"), + ("Author 5", "author5@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="Trouble Maker", + submitter_email="", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 2", "author2@example.net"), + ], + ) + + def make_draft_submission( + self, year, month, day, submitter_name, submitter_email, author_nameaddrs + ): + + authors = [] + for name, addr in author_nameaddrs: + person = Person.objects.filter(name=name).first() + if not person: + person = EmailFactory(person__name=name, address=addr).person + elif not Email.objects.filter(address=addr).exists(): + EmailFactory(person=person, address=addr) + authors.append(person) + + submission = SubmissionFactory( + submission_date=datetime.date(year, month, day), + submitter_name=submitter_name, + submitter_email=submitter_email, + state_id="posted", + ) + submission.authors = [ + { + "name": f"{name}", + "email": f"{addr}", + "affiliation": "", + "country": "", + "errors": [], + } + for name, addr in author_nameaddrs + ] + + submission.save() + IndividualDraftFactory(submission=submission, authors=authors) + + def test_authors_by_year(self): + authors2020 = authors_by_year(2020) + self.assertEqual( + authors2020, + set( + [ + "author0@example.net", + "author0@example.com", + "author1@example.net", + "author3@example.net", + ] + ), + ) + authors2021 = authors_by_year(2021) + self.assertEqual( + authors2021, + set( + [ + "author0@example.net", + "author0@example.com", + "author2@example.net", + "author3@example.net", + "author4@example.net", + "author5@example.net", + ] + ), + ) + + def test_submitters_by_year(self): + sub2020 = submitters_by_year(2020) + self.assertEqual( + sub2020, + set( + [ + "author0@example.net", + "author3@example.net", + "notanauthor0@example.net", + ] + ), + ) + sub2021 = submitters_by_year(2021) + self.assertEqual( + sub2021, set(["author0@example.com", "notanauthor2@example.net"]) + ) + + def test_unique_people(self): + persons, addrs = unique_people( + [ + "notanauthor0@example.com", + "author0@example.net", + "author0@example.com", + "author1@example.net", + "notanauthor0@example.com", + ] + ) + self.assertEqual(addrs, set(["notanauthor0@example.com"])) + self.assertEqual( + set(persons), set(Person.objects.filter(name__in=("Author 0", "Author 1"))) + ) + self.assertEqual(len(persons) + len(addrs), 3) From 7dff4a7af8844a22a302a579ace59c8372a5348b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 8 Jun 2026 16:22:23 -0300 Subject: [PATCH 151/188] ci: remove strategy from dt manifests (#11004) --- k8s/auth.yaml | 2 -- k8s/datatracker.yaml | 2 -- 2 files changed, 4 deletions(-) diff --git a/k8s/auth.yaml b/k8s/auth.yaml index 6e63001e02b..ef8c259933b 100644 --- a/k8s/auth.yaml +++ b/k8s/auth.yaml @@ -8,8 +8,6 @@ spec: selector: matchLabels: app: auth - strategy: - type: DEPLOY_STRATEGY template: metadata: labels: diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index af2bb6295c9..5183893bc8c 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -8,8 +8,6 @@ spec: selector: matchLabels: app: datatracker - strategy: - type: DEPLOY_STRATEGY template: metadata: labels: From 7e3e3cb503aa9ec2f757cabab85c944771ade8f9 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 9 Jun 2026 18:51:19 -0500 Subject: [PATCH 152/188] fix: tweak logging settings to avoid spurious keyerrors (#11009) --- ietf/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ietf/settings.py b/ietf/settings.py index 95f2ffefd76..1286e70e75c 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -329,7 +329,8 @@ def skip_unreadable_post(record): "formatters": { "django.server": { "()": "django.utils.log.ServerFormatter", - "format": "[%(server_time)s] %(message)s", + "format": "[{server_time}] {message}", + "style": "{", }, "plain": { "style": "{", From c342bef7938e2ba56b23984c17b3af740c52ac8e Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 9 Jun 2026 19:08:23 -0500 Subject: [PATCH 153/188] fix: improved source and keywords for rfc json (#11008) --- ietf/doc/tests_utils_rfc_json.py | 58 ++++++++++++++++++++++++++++++-- ietf/doc/utils_rfc_json.py | 20 +++++------ 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/ietf/doc/tests_utils_rfc_json.py b/ietf/doc/tests_utils_rfc_json.py index b94fc2cbea1..075b50a24f3 100644 --- a/ietf/doc/tests_utils_rfc_json.py +++ b/ietf/doc/tests_utils_rfc_json.py @@ -13,6 +13,7 @@ PublishedRfcDocEventFactory, RfcAuthorFactory, RfcFactory, + RgRfcFactory, WgRfcFactory, ) from ietf.doc.models import RelatedDocument @@ -245,6 +246,19 @@ def test_april_first_date_format(self): self.assertEqual(data["pub_date"], "1 April 2020") + def test_empty_keywords_filtered(self): + """Empty-string keywords are stripped from the keywords list.""" + rfc = PublishedRfcDocEventFactory( + doc=WgRfcFactory(keywords=["foo", "", "bar"]), + ).doc + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["keywords"], ["foo", "bar"]) + def test_non_april_first_april_date(self): """An April publication that is NOT in the April Fools list gets 'April YYYY'.""" rfc = PublishedRfcDocEventFactory( @@ -260,7 +274,7 @@ def test_non_april_first_april_date(self): self.assertEqual(data["pub_date"], "April 2020") def test_source_ietf_wg(self): - """IETF-stream WG RFC: source is 'acronym (area)'.""" + """IETF-stream WG RFC: source is the group's full name.""" area = GroupFactory(type_id="area") wg = GroupFactory(type_id="wg", parent=area) rfc = PublishedRfcDocEventFactory( @@ -272,7 +286,7 @@ def test_source_ietf_wg(self): generate_rfc_json(rfc.rfc_number) data = _read_json(rfc.rfc_number) - self.assertEqual(data["source"], f"{wg.acronym} ({area.acronym})") + self.assertEqual(data["source"], wg.name) def test_source_ietf_no_wg(self): """IETF-stream individual RFC (group acronym 'none'): source is 'IETF - NON WORKING GROUP'.""" @@ -293,6 +307,22 @@ def test_source_ietf_no_wg(self): self.assertEqual(data["source"], "IETF - NON WORKING GROUP") + def test_source_ietf_area(self): + """IETF-stream RFC with area-type group: source is 'IETF - NON WORKING GROUP'.""" + area = GroupFactory(type_id="area") + area.parent = GroupFactory() + area.save() + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(group=area, stream_id="ietf"), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IETF - NON WORKING GROUP") + def test_source_iab(self): """IAB-stream RFC: source is 'IAB'.""" rfc = PublishedRfcDocEventFactory( @@ -319,6 +349,30 @@ def test_source_ise(self): self.assertEqual(data["source"], "INDEPENDENT") + def test_source_irtf_rg(self): + """IRTF-stream RG RFC: source is the group's full name.""" + rfc = PublishedRfcDocEventFactory(doc=RgRfcFactory()).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], rfc.group.name) + + def test_source_irtf_no_rg(self): + """IRTF-stream RFC with no specific RG (group acronym 'none'): source is 'IRTF'.""" + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(stream_id="irtf", group=GroupFactory(acronym="none")), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IRTF") + def test_pub_levels_passed_in(self): """When pub_levels is passed in, get_publication_std_levels() is not called.""" rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc diff --git a/ietf/doc/utils_rfc_json.py b/ietf/doc/utils_rfc_json.py index 6030a7064c5..af916e0d7fc 100644 --- a/ietf/doc/utils_rfc_json.py +++ b/ietf/doc/utils_rfc_json.py @@ -105,7 +105,6 @@ def generate_rfc_json(rfc_number: int, *, pub_levels=None) -> None: stream_slug = rfc.stream.slug group_acronym = rfc.group.acronym - area_acronym = None if stream_slug == "ietf": if rfc.group.parent is None: assertion("rfc.group.parent is not None") @@ -113,23 +112,22 @@ def generate_rfc_json(rfc_number: int, *, pub_levels=None) -> None: f"Malformed document object encountered for rfc{rfc_number}. Aborting update of rfc{rfc_number}.json" ) return - else: - area_acronym = rfc.group.parent.acronym if stream_slug == "ise": source = "INDEPENDENT" elif stream_slug == "iab": source = "IAB" elif stream_slug == "ietf" and ( - group_acronym in ("none", "gen") or not area_acronym + group_acronym == "none" or rfc.group.type_id == "area" ): source = "IETF - NON WORKING GROUP" - elif group_acronym not in ("none", ""): - source = group_acronym - if stream_slug == "ietf" and area_acronym: - source += f" ({area_acronym})" - elif stream_slug: - source += f" ({stream_slug})" + elif stream_slug == "irtf": + if group_acronym == "none": + source = "IRTF" + else: + source = rfc.group.name + elif group_acronym not in ("none", "") and stream_slug in ["ietf", "editorial"]: + source = rfc.group.name elif stream_slug: source = "Legacy" if stream_slug == "legacy" else stream_slug.upper() else: @@ -207,7 +205,7 @@ def _rfc_list(qs, attr): "source": source, "abstract": rfc.abstract, "pub_date": pub_date, - "keywords": rfc.keywords, + "keywords": [kw for kw in rfc.keywords if kw], "obsoletes": obsoletes, "obsoleted_by": obsoleted_by, "updates": updates, From 1cccb088947cab50a5b8c837ed584f2396de2cdd Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 9 Jun 2026 19:09:36 -0500 Subject: [PATCH 154/188] test: make check for April 1 resilient across line-breaks (#11006) * test: make check for April 1 resilient across line-breaks * test: be resilient against faker creating a 63 character long title --- ietf/sync/tests_rfcindex.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index 2b70924db39..1a45d5d9cf4 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -139,7 +139,10 @@ def test_create_rfc_txt_index(self, mock_save_blob, mock_save_file): f"{self.april_fools_rfc.rfc_number} {self.april_fools_rfc.title}", stripped_contents, ) - self.assertIn("1 April 2020", contents) # from the April 1 RFC + # "1 April 2020" may be split across a line wrap (e.g. "1 April\n 2020") + # when the randomly-generated title is long enough to push the date off the line. + # assertRegex handles both wrapped and non-wrapped cases explicitly. + self.assertRegex(contents, r"1\s+April\s+2020") # from the April 1 RFC self.assertIn( f"{self.rfc.rfc_number} {self.rfc.title}", stripped_contents, From 28a5e967725b8569bde7ca09821b182a5bcbe9b5 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 10 Jun 2026 12:08:45 -0300 Subject: [PATCH 155/188] chore(dev): fix nginx config (#11014) keepalive 0 is not supported by the version of nginx in the dev container --- docker/configs/nginx-proxy.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/configs/nginx-proxy.conf b/docker/configs/nginx-proxy.conf index a02ab2ff06a..0a9dde04eb4 100644 --- a/docker/configs/nginx-proxy.conf +++ b/docker/configs/nginx-proxy.conf @@ -1,6 +1,7 @@ upstream datatracker_backend { server 127.0.0.1:8001; - keepalive 0; # default = 32 since nginx 1.29.7 +# Uncomment when changing to nginx 1.29.7 or later. +# keepalive 0; # default = 32 since nginx 1.29.7 } server { From 6b185be2961952e25a3ec226310593a4544a4a10 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 10 Jun 2026 18:39:32 -0500 Subject: [PATCH 156/188] fix: sort the search results document column correctly (#11015) * fix: sort the group document's document column correcty * test: confirm column sort information is present --- ietf/group/tests_info.py | 9 ++++++++- ietf/templates/doc/search/search_result_row.html | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 3f24e2e3d6b..97ec7ebdb15 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -27,7 +27,7 @@ from ietf.community.models import CommunityList from ietf.community.utils import reset_name_contains_index_for_rule -from ietf.doc.factories import WgDraftFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory +from ietf.doc.factories import WgDraftFactory, WgRfcFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory from ietf.doc.models import Document, DocEvent, State from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils_charter import charter_name_for_group @@ -396,6 +396,7 @@ def test_group_documents(self): draft7 = WgDraftFactory(group=group) draft7.set_state(State.objects.get(type='draft', slug='expired')) draft7.set_state(State.objects.get(type='draft-stream-%s' % draft7.stream_id, slug='dead')) # Expired WG draft, marked as dead + rfc = WgRfcFactory(group=group) clist = CommunityList.objects.get(group=group) related_docs_rule = clist.searchrule_set.get(rule_type='name_contains') @@ -426,6 +427,12 @@ def test_group_documents(self): q = PyQuery(r.content) self.assertTrue(any([draft2.name in x.attrib['href'] for x in q('table td a.track-untrack-doc')])) + # RFC rows must use the RFC number as the sort key so that numeric sort + # is not disrupted by the page-count text that precedes the name in the cell. + # Draft rows must use the document name. + self.assertTrue(q(f'td.doc[data-sort-number="{rfc.rfc_number}"]')) + self.assertTrue(q(f'td.doc[data-sort-number="{draft.name}"]')) + # Let's also check the IRTF stream rg = GroupFactory(type_id='rg') setup_default_community_list_for_group(rg) diff --git a/ietf/templates/doc/search/search_result_row.html b/ietf/templates/doc/search/search_result_row.html index 476c81f5986..cb50a1b214e 100644 --- a/ietf/templates/doc/search/search_result_row.html +++ b/ietf/templates/doc/search/search_result_row.html @@ -45,7 +45,7 @@ {% endfor %} - + {% if doc.pages %}{{ doc.pages }} page{{ doc.pages|pluralize }}{% endif %}
    From f77514e98f76d9cc263c829f616bc6822864b01b Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 10 Jun 2026 18:41:03 -0500 Subject: [PATCH 157/188] fix: correct sorting in nomcom feedback tables (#11016) --- ietf/nomcom/tests.py | 14 ++++++++++++++ ietf/templates/nomcom/view_feedback.html | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index 210788ce079..28152ef79b1 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -1640,6 +1640,20 @@ def test_feedback_topic_badges(self): q = PyQuery(response.content) self.assertEqual( len(q('.text-bg-success')), 0 ) + def test_feedback_index_sort_keys(self): + url = reverse('ietf.nomcom.views.view_feedback', kwargs={'year': self.nc.year()}) + login_testing_unauthorized(self, self.member.user.username, url) + provide_private_key_to_test_client(self) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + q = PyQuery(response.content) + # Feedback count cells must carry a numeric data-sort-number so that + # a "New" badge appearing before the count doesn't corrupt the sort key. + sort_cells = q('td[data-sort-number]') + self.assertTrue(len(sort_cells) > 0) + for cell in sort_cells.items(): + self.assertRegex(cell.attr('data-sort-number'), r'^\d+$') + class NewActiveNomComTests(TestCase): def setUp(self): diff --git a/ietf/templates/nomcom/view_feedback.html b/ietf/templates/nomcom/view_feedback.html index 93d61b7f420..0758329d606 100644 --- a/ietf/templates/nomcom/view_feedback.html +++ b/ietf/templates/nomcom/view_feedback.html @@ -43,7 +43,7 @@

    Declined each nominated position

     {% for fbtype_name, fbtype_count, fbtype_newflag in fb_dict.feedback %} - + {% if fbtype_newflag %}New{% endif %} {{ fbtype_count }} @@ -82,7 +82,7 @@

    Feedback related to topics

    {% for fbtype_name, fbtype_count, fbtype_newflag in fb_dict.feedback %} - + {% if fbtype_newflag %}New{% endif %} {{ fbtype_count }} From 65dcdc942956d5a9dc21bf7029664b04d49c8d00 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 10 Jun 2026 20:42:58 -0300 Subject: [PATCH 158/188] refactor: use correct red bucket paths by default (#11007) * refactor: RFCINDEX_INPUT_PATH dflt -> prod value * refactor: RFCINDEX_OUTPUT_PATH dflt -> prod value * ci: only change RFCINDEX_* paths if set * refactor: clearer settings * test: update / fix / add tests --- ietf/settings.py | 4 +++ ietf/sync/rfcindex.py | 19 ++++++++------ ietf/sync/tests_rfcindex.py | 50 +++++++++++++++++++++++++++++-------- k8s/settings_local.py | 6 +++-- 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index 1286e70e75c..d509a877c61 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -977,6 +977,10 @@ def skip_unreadable_post(record): ) RFC_FILE_TYPES = IDSUBMIT_FILE_TYPES +# Paths in the red bucket +RFCINDEX_INPUT_PATH = "other/" +RFCINDEX_OUTPUT_PATH = "other/" + IDSUBMIT_MAX_DRAFT_SIZE = { 'txt': 2*1024*1024, # Max size of txt draft file in bytes 'xml': 3*1024*1024, # Max size of xml draft file in bytes diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index be55a6866e3..f47974f9002 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -48,9 +48,17 @@ def errata_url(rfc: Document): return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}") +def red_bucket_input_path(filename: str) -> str: + return str(Path(settings.RFCINDEX_INPUT_PATH) / filename) + + +def red_bucket_output_path(filename: str) -> str: + return str(Path(settings.RFCINDEX_OUTPUT_PATH) / filename) + + def save_to_red_bucket(filename: str, content: str | bytes): red_bucket = storages["red_bucket"] - bucket_path = str(Path(getattr(settings, "RFCINDEX_OUTPUT_PATH", "")) / filename) + bucket_path = red_bucket_output_path(filename) if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True): # Django 4.2's FileSystemStorage does not support allow_overwrite. red_bucket.delete(bucket_path) @@ -87,8 +95,7 @@ class UnusableRfcNumber: def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: - FILENAME = "unusable-rfc-numbers.json" - bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + bucket_path = red_bucket_input_path("unusable-rfc-numbers.json") try: with storages["red_bucket"].open(bucket_path) as urn_file: records = json.load(urn_file) @@ -115,8 +122,7 @@ def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: def get_april1_rfc_numbers() -> Container[int]: - FILENAME = "april-first-rfc-numbers.json" - bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + bucket_path = red_bucket_input_path("april-first-rfc-numbers.json") try: with storages["red_bucket"].open(bucket_path) as urn_file: records = json.load(urn_file) @@ -139,8 +145,7 @@ def get_april1_rfc_numbers() -> Container[int]: def get_publication_std_levels() -> dict[int, StdLevelName]: - FILENAME = "publication-std-levels.json" - bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + bucket_path = red_bucket_input_path("publication-std-levels.json") values: dict[int, StdLevelName] = {} try: with storages["red_bucket"].open(bucket_path) as urn_file: diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index 1a45d5d9cf4..226b41af946 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -28,9 +28,11 @@ get_april1_rfc_numbers, get_publication_std_levels, get_unusable_rfc_numbers, + red_bucket_input_path, + red_bucket_output_path, + save_to_filesystem, save_to_red_bucket, subseries_text_line, - save_to_filesystem, ) from ietf.utils.test_utils import TestCase @@ -401,25 +403,47 @@ def test_create_fyi_txt_index(self, mock_save_blob, mock_save_file): ) +@override_settings(RFCINDEX_INPUT_PATH="input/", RFCINDEX_OUTPUT_PATH="output/") class HelperTests(TestCase): + INPUT_PATH = "input" + OUTPUT_PATH = "output" + def test_format_rfc_number(self): self.assertEqual(format_rfc_number(10), "10") with override_settings(RFCINDEX_MATCH_LEGACY_XML=True): self.assertEqual(format_rfc_number(10), "0010") + def test_red_bucket_input_path(self): + with override_settings(RFCINDEX_INPUT_PATH="bar"): + self.assertEqual(red_bucket_input_path("foo"), "bar/foo") + with override_settings(RFCINDEX_INPUT_PATH="bar/"): + self.assertEqual(red_bucket_input_path("foo"), "bar/foo") + + def test_red_bucket_output_path(self): + self.assertEqual(red_bucket_input_path("foo"), f"{self.INPUT_PATH}/foo") + with override_settings(RFCINDEX_OUTPUT_PATH="bar"): + self.assertEqual(red_bucket_output_path("foo"), "bar/foo") + with override_settings(RFCINDEX_OUTPUT_PATH="bar/"): + self.assertEqual(red_bucket_output_path("foo"), "bar/foo") + def test_save_to_red_bucket(self): red_bucket = storages["red_bucket"] with override_settings(RFCINDEX_DELETE_THEN_WRITE=False): save_to_red_bucket("test", "contents \U0001f600") # Read as binary and explicitly decode to confirm encoding - with red_bucket.open("test", "rb") as f: + with red_bucket.open(f"{self.OUTPUT_PATH}/test", "rb") as f: self.assertEqual(f.read().decode("utf-8"), "contents \U0001f600") with override_settings(RFCINDEX_DELETE_THEN_WRITE=True): save_to_red_bucket("test", "new contents \U0001fae0".encode("utf-8")) # Read as binary and explicitly decode to confirm encoding - with red_bucket.open("test", "rb") as f: + with red_bucket.open(f"{self.OUTPUT_PATH}/test", "rb") as f: self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0") - red_bucket.delete("test") # clean up like a good child + red_bucket.delete(f"{self.OUTPUT_PATH}/test") # clean up like a good child + # check that we can override the path + with override_settings(RFCINDEX_OUTPUT_PATH="fruit"): + save_to_red_bucket("test", "content") + self.assertTrue(red_bucket.exists("fruit/test")) + red_bucket.delete("fruit/test") # clean up like a good child def test_save_to_filesystem(self): rfc_path = Path(settings.RFC_PATH) @@ -445,30 +469,36 @@ def test_get_unusable_rfc_numbers_raises(self): with self.assertRaises(FileNotFoundError): get_unusable_rfc_numbers() red_bucket = storages["red_bucket"] - red_bucket.save("unusable-rfc-numbers.json", ContentFile("not json")) + red_bucket.save( + f"{self.INPUT_PATH}/unusable-rfc-numbers.json", ContentFile("not json") + ) with self.assertRaises(json.JSONDecodeError): get_unusable_rfc_numbers() - red_bucket.delete("unusable-rfc-numbers.json") + red_bucket.delete(f"{self.INPUT_PATH}/unusable-rfc-numbers.json") def test_get_april1_rfc_numbers_raises(self): """get_april1_rfc_numbers should bail on errors""" with self.assertRaises(FileNotFoundError): get_april1_rfc_numbers() red_bucket = storages["red_bucket"] - red_bucket.save("april-first-rfc-numbers.json", ContentFile("not json")) + red_bucket.save( + f"{self.INPUT_PATH}/april-first-rfc-numbers.json", ContentFile("not json") + ) with self.assertRaises(json.JSONDecodeError): get_april1_rfc_numbers() - red_bucket.delete("april-first-rfc-numbers.json") + red_bucket.delete(f"{self.INPUT_PATH}/april-first-rfc-numbers.json") def test_get_publication_std_levels_raises(self): """get_publication_std_levels should bail on errors""" with self.assertRaises(FileNotFoundError): get_publication_std_levels() red_bucket = storages["red_bucket"] - red_bucket.save("publication-std-levels.json", ContentFile("not json")) + red_bucket.save( + f"{self.INPUT_PATH}/publication-std-levels.json", ContentFile("not json") + ) with self.assertRaises(json.JSONDecodeError): get_publication_std_levels() - red_bucket.delete("publication-std-levels.json") + red_bucket.delete(f"{self.INPUT_PATH}/publication-std-levels.json") def test_subseries_text_line(self): text = "foobar" diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 20c5252ff03..5dc31bac0e5 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -472,8 +472,10 @@ def _multiline_to_list(s): ), } RFCINDEX_DELETE_THEN_WRITE = False # S3Storage allows file_overwrite by default -RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/") -RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH", "") +if "DATATRACKER_RFCINDEX_OUTPUT_PATH" in os.environ: + RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH") +if "DATATRACKER_RFCINDEX_INPUT_PATH" in os.environ: + RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH") # Configure the blobdb app for artifact storage _blobdb_replication_enabled = ( From e79b0575cd36273837a2e6f2d82847889cf34541 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 11 Jun 2026 12:27:41 -0500 Subject: [PATCH 159/188] fix: include draft revision in rfc json (#11024) --- ietf/doc/tests_utils_rfc_json.py | 14 ++++++++++++++ ietf/doc/utils_rfc_json.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/ietf/doc/tests_utils_rfc_json.py b/ietf/doc/tests_utils_rfc_json.py index 075b50a24f3..ec3cd85893f 100644 --- a/ietf/doc/tests_utils_rfc_json.py +++ b/ietf/doc/tests_utils_rfc_json.py @@ -14,6 +14,7 @@ RfcAuthorFactory, RfcFactory, RgRfcFactory, + WgDraftFactory, WgRfcFactory, ) from ietf.doc.models import RelatedDocument @@ -199,6 +200,19 @@ def test_errata_url_set_when_errata_exist(self): f"https://www.rfc-editor.org/errata/rfc{rfc.rfc_number}", ) + def test_draft_field_includes_revision(self): + """draft field is '-' when the RFC originated from a draft.""" + draft = WgDraftFactory(rev="07") + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc) + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["draft"], f"{draft.name}-07") + def test_errata_url_none_when_no_errata(self): """errata_url is None when errata.json has no entries for the RFC.""" rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc diff --git a/ietf/doc/utils_rfc_json.py b/ietf/doc/utils_rfc_json.py index af916e0d7fc..1f134556869 100644 --- a/ietf/doc/utils_rfc_json.py +++ b/ietf/doc/utils_rfc_json.py @@ -55,7 +55,7 @@ def generate_rfc_json(rfc_number: int, *, pub_levels=None) -> None: # draft name draft_doc = rfc.came_from_draft() - draft = draft_doc.name if draft_doc else None + draft = f"{draft_doc.name}-{draft_doc.rev}" if draft_doc else None # authors: ordered list of display strings authors = [] From 7666ae23c88671eb8f72b13ac72156c7046d5039 Mon Sep 17 00:00:00 2001 From: NGPixel Date: Thu, 11 Jun 2026 18:19:52 -0400 Subject: [PATCH 160/188] ci: fix lock-threads workflow --- .github/workflows/lock-threads.yml | 2 +- .github/workflows/tests-az.yml | 109 ----------------------------- 2 files changed, 1 insertion(+), 110 deletions(-) delete mode 100644 .github/workflows/tests-az.yml diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml index 22652dab889..ab2c1a7cf6b 100644 --- a/.github/workflows/lock-threads.yml +++ b/.github/workflows/lock-threads.yml @@ -16,7 +16,7 @@ jobs: action: runs-on: ubuntu-latest steps: - - uses: ietf-tools/lock-threads@v3.1.1 + - uses: dessant/lock-threads@v6 with: github-token: ${{ github.token }} issue-inactive-days: 7 diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml deleted file mode 100644 index 833ca89befd..00000000000 --- a/.github/workflows/tests-az.yml +++ /dev/null @@ -1,109 +0,0 @@ -name: Tests (Azure Test) - -on: - workflow_dispatch: - -jobs: - main: - name: Run Tests on Azure temp VM - runs-on: ubuntu-latest - - permissions: - contents: read - - steps: - - name: Launch VM on Azure - id: azlaunch - run: | - echo "Authenticating to Azure..." - az login --service-principal -u ${{ secrets.AZ_TESTS_APP_ID }} -p ${{ secrets.AZ_TESTS_PWD }} --tenant ${{ secrets.AZ_TESTS_TENANT_ID }} - echo "Creating VM..." - vminfo=$(az vm create \ - --resource-group ghaDatatrackerTests \ - --name tmpGhaVM2 \ - --image Ubuntu2204 \ - --admin-username azureuser \ - --generate-ssh-keys \ - --priority Spot \ - --size Standard_D4as_v5 \ - --max-price -1 \ - --os-disk-size-gb 30 \ - --eviction-policy Delete \ - --nic-delete-option Delete \ - --output tsv \ - --query "publicIpAddress") - echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT" - echo "VM Public IP: $vminfo" - cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key - ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts - - - name: Remote SSH into VM - uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - host: ${{ steps.azlaunch.outputs.ipaddr }} - port: 22 - username: azureuser - command_timeout: 60m - key_path: ${{ github.workspace }}/prvkey.key - envs: GITHUB_TOKEN - script_stop: true - script: | - export DEBIAN_FRONTEND=noninteractive - lsb_release -a - sudo apt-get update - sudo apt-get upgrade -y - - echo "Installing Docker..." - curl -fsSL https://get.docker.com -o get-docker.sh - sudo sh get-docker.sh - - echo "Starting Containers..." - sudo docker network create dtnet - sudo docker run -d --name db --network=dtnet ghcr.io/ietf-tools/datatracker-db:latest & - sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:latest sleep infinity & - wait - - echo "Cloning datatracker repo..." - sudo docker exec app git clone --depth=1 https://github.com/ietf-tools/datatracker.git . - echo "Prepare tests..." - sudo docker exec app chmod +x ./dev/tests/prepare.sh - sudo docker exec app sh ./dev/tests/prepare.sh - echo "Running checks..." - sudo docker exec app ietf/manage.py check - sudo docker exec app ietf/manage.py migrate --fake-initial - echo "Running tests..." - sudo docker exec app ietf/manage.py test -v2 --validate-html-harder --settings=settings_test - - - name: Destroy VM + resources - if: always() - shell: pwsh - run: | - echo "Destroying VM..." - az vm delete -g ghaDatatrackerTests -n tmpGhaVM2 --yes --force-deletion true - - $resourceOrderRemovalOrder = [ordered]@{ - "Microsoft.Compute/virtualMachines" = 0 - "Microsoft.Compute/disks" = 1 - "Microsoft.Network/networkInterfaces" = 2 - "Microsoft.Network/publicIpAddresses" = 3 - "Microsoft.Network/networkSecurityGroups" = 4 - "Microsoft.Network/virtualNetworks" = 5 - } - echo "Fetching remaining resources..." - $resources = az resource list --resource-group ghaDatatrackerTests | ConvertFrom-Json - - $orderedResources = $resources - | Sort-Object @{ - Expression = {$resourceOrderRemovalOrder[$_.type]} - Descending = $False - } - - echo "Deleting remaining resources..." - $orderedResources | ForEach-Object { - az resource delete --resource-group ghaDatatrackerTests --ids $_.id --verbose - } - - echo "Logout from Azure..." - az logout From 65e7d51cd4ada4c6c817b9348f5671f45633f899 Mon Sep 17 00:00:00 2001 From: NGPixel Date: Thu, 11 Jun 2026 18:22:12 -0400 Subject: [PATCH 161/188] ci: fix lock-threads workflow --- .github/workflows/lock-threads.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml index ab2c1a7cf6b..b90300c09b2 100644 --- a/.github/workflows/lock-threads.yml +++ b/.github/workflows/lock-threads.yml @@ -19,6 +19,7 @@ jobs: - uses: dessant/lock-threads@v6 with: github-token: ${{ github.token }} + process-only: 'issues, prs' issue-inactive-days: 7 pr-inactive-days: 3 log-output: true From ecebc399dda88494a04ccb59fd8ec24fa4473a4d Mon Sep 17 00:00:00 2001 From: NGPixel Date: Thu, 11 Jun 2026 22:23:03 -0400 Subject: [PATCH 162/188] ci: refactor build + deploy workflows --- .github/workflows/build.yml | 349 +++--------------- .github/workflows/ci-run-tests.yml | 2 +- .github/workflows/release.yml | 209 +++++++++++ .github/workflows/reusable-build.yml | 226 ++++++++++++ .../{tests.yml => reusable-tests.yml} | 0 5 files changed, 489 insertions(+), 297 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/reusable-build.yml rename .github/workflows/{tests.yml => reusable-tests.yml} (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a89bac46e74..2c449b53ce8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,10 +1,7 @@ -name: Build and Release -run-name: ${{ github.ref_name == 'release' && '[Prod]' || '[Dev]' }} Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} +name: Build Dev and Deploy +run-name: Dev Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} on: - push: - branches: [release] - workflow_dispatch: inputs: deploy: @@ -14,24 +11,28 @@ on: type: choice options: - Skip - - Staging Only - - Staging + Prod - dev: - description: 'Deploy to Dev' - default: true + - Dev + - Staging + deployStrategy: + description: 'Deploy Strategy' + default: 'Recreate' required: true - type: boolean + type: choice + options: + - Recreate + - RollingUpdate + - Current devNoDbRefresh: description: 'Dev Disable Daily DB Refresh' default: false required: true type: boolean - skiptests: + skipTests: description: 'Skip Tests' default: false required: true type: boolean - skiparm: + skipArmBuild: description: 'Skip ARM64 Build' default: false required: true @@ -56,14 +57,12 @@ jobs: # PREPARE # ----------------------------------------------------------------- prepare: - name: Prepare Release + name: Prepare runs-on: ubuntu-latest outputs: - should_deploy: ${{ steps.buildvars.outputs.should_deploy }} - pkg_version: ${{ steps.buildvars.outputs.pkg_version }} - from_tag: ${{ steps.semver.outputs.nextStrict }} - to_tag: ${{ steps.semver.outputs.current }} - base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} + buildVersion: ${{ steps.buildvars.outputs.buildVersion }} + previousVersion: ${{ steps.semver.outputs.current }} + baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} steps: - uses: actions/checkout@v6 @@ -71,19 +70,8 @@ jobs: fetch-depth: 1 fetch-tags: false - - name: Get Next Version (Prod) - if: ${{ github.ref_name == 'release' }} - id: semver - uses: ietf-tools/semver-action@v1 - with: - token: ${{ github.token }} - branch: release - skipInvalidTags: true - patchList: fix, bugfix, perf, refactor, test, tests, chore - - name: Get Dev Version - if: ${{ github.ref_name != 'release' }} - id: semverdev + id: semver uses: ietf-tools/semver-action@v1 with: token: ${{ github.token }} @@ -91,260 +79,53 @@ jobs: skipInvalidTags: true noVersionBumpBehavior: 'current' noNewCommitBehavior: 'current' - - - name: Set Release Flag - if: ${{ github.ref_name == 'release' }} - run: | - echo "IS_RELEASE=true" >> $GITHUB_ENV - - - name: Create Draft Release - uses: ncipollo/release-action@v1.21.0 - if: ${{ github.ref_name == 'release' }} - with: - prerelease: true - draft: false - commit: ${{ github.sha }} - tag: ${{ steps.semver.outputs.nextStrict }} - name: ${{ steps.semver.outputs.nextStrict }} - body: '*pending*' - token: ${{ secrets.GITHUB_TOKEN }} - name: Set Build Variables id: buildvars run: | - if [[ $IS_RELEASE ]]; then - echo "Using AUTO SEMVER mode: ${{ steps.semver.outputs.nextStrict }}" - echo "should_deploy=true" >> $GITHUB_OUTPUT - echo "pkg_version=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT - echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME" - else - echo "Using TEST mode: ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" - echo "should_deploy=false" >> $GITHUB_OUTPUT - echo "pkg_version=${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT - echo "::notice::Non-production build ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME" - fi + SHORT_SHA="${GITHUB_SHA:0:7}" + echo "Will set build version to: ${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA" + echo "buildVersion=${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA" >> $GITHUB_OUTPUT + echo "::notice::Non-production build ${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA created using branch $GITHUB_REF_NAME" - name: Get Base Image Target Version id: baseimgversion run: | - echo "base_image_version=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + echo "baseImageVersion=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT # ----------------------------------------------------------------- # TESTS # ----------------------------------------------------------------- - tests: name: Run Tests - uses: ./.github/workflows/tests.yml - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} + uses: ./.github/workflows/reusable-tests.yml + if: ${{ inputs.skipTests == 'false' }} needs: [prepare] secrets: inherit with: - ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }} + ignoreLowerCoverage: ${{ inputs.ignoreLowerCoverage == 'true' }} skipSelenium: true - targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }} + targetBaseVersion: ${{ needs.prepare.outputs.baseImageVersion }} # ----------------------------------------------------------------- - # RELEASE + # BUILD IMAGE # ----------------------------------------------------------------- - release: - name: Make Release + build: + name: Build Image + uses: ./.github/workflows/reusable-build.yml if: ${{ !failure() && !cancelled() }} needs: [tests, prepare] - runs-on: - group: hperf-8c32r permissions: contents: write packages: write - env: - SHOULD_DEPLOY: ${{needs.prepare.outputs.should_deploy}} - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - FROM_TAG: ${{needs.prepare.outputs.from_tag}} - TO_TAG: ${{needs.prepare.outputs.to_tag}} - TARGET_BASE: ${{needs.prepare.outputs.base_image_version}} - - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 1 - fetch-tags: false - - - name: Setup Node.js environment - uses: actions/setup-node@v6 - with: - node-version: 18.x - - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - - name: Setup AWS CLI - uses: unfor19/install-aws-cli-action@v1 - with: - version: 2.22.35 - - - name: Download a Coverage Results - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v8.0.1 - with: - name: coverage - - - name: Make Release Build - env: - DEBIAN_FRONTEND: noninteractive - BROWSERSLIST_IGNORE_OLD_DATA: 1 - run: | - echo "PKG_VERSION: $PKG_VERSION" - echo "GITHUB_SHA: $GITHUB_SHA" - echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" - echo "Running frontend build script..." - echo "Compiling native node packages..." - yarn rebuild - echo "Packaging static assets..." - yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/ - yarn legacy:build - echo "Setting version $PKG_VERSION..." - sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py - sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py - sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py - - - name: Set Production Flags - if: ${{ env.SHOULD_DEPLOY == 'true' }} - run: | - echo "Setting production flags in settings.py..." - sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py - - - name: Make Release Tarball - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "Build release tarball..." - mkdir -p /home/runner/work/release - tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . - - - name: Collect + Push Statics - env: - DEBIAN_FRONTEND: noninteractive - AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} - AWS_DEFAULT_REGION: auto - AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} - run: | - echo "Collecting statics..." - echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}" - docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh - echo "Pushing statics..." - cd static - aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors - - - name: Augment dockerignore for docker image build - env: - DEBIAN_FRONTEND: noninteractive - run: | - cat >> .dockerignore <> $GITHUB_ENV - - - name: Build Images - uses: docker/build-push-action@v7 - env: - DOCKER_BUILD_SUMMARY: false - with: - context: . - file: dev/build/Dockerfile - platforms: ${{ github.event.inputs.skiparm == 'true' && 'linux/amd64' || 'linux/amd64,linux/arm64' }} - push: true - tags: | - ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} - ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} - - - name: Update CHANGELOG - id: changelog - uses: Requarks/changelog-action@v1 - if: ${{ env.SHOULD_DEPLOY == 'true' }} - with: - token: ${{ github.token }} - fromTag: ${{ env.FROM_TAG }} - toTag: ${{ env.TO_TAG }} - writeToFile: false - - - name: Download Coverage Results - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v8.0.1 - with: - name: coverage - - - name: Prepare Coverage Action - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - working-directory: ./dev/coverage-action - run: npm install - - - name: Process Coverage Stats + Chart - id: covprocess - uses: ./dev/coverage-action/ - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - with: - token: ${{ github.token }} - tokenCommon: ${{ secrets.GH_COMMON_TOKEN }} - repoCommon: common - version: ${{needs.prepare.outputs.pkg_version}} - changelog: ${{ steps.changelog.outputs.changes }} - summary: '' - coverageResultsPath: coverage.json - histCoveragePath: historical-coverage.json - - - name: Create Release - uses: ncipollo/release-action@v1.21.0 - if: ${{ env.SHOULD_DEPLOY == 'true' }} - with: - allowUpdates: true - makeLatest: true - draft: false - tag: ${{ env.PKG_VERSION }} - name: ${{ env.PKG_VERSION }} - body: ${{ steps.covprocess.outputs.changelog }} - artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Update Baseline Coverage - uses: ncipollo/release-action@v1.21.0 - if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }} - with: - allowUpdates: true - tag: baseline - omitBodyDuringUpdate: true - omitNameDuringUpdate: true - omitPrereleaseDuringUpdate: true - replacesArtifacts: true - artifacts: "coverage.json" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Upload Build Artifacts - uses: actions/upload-artifact@v7 - with: - name: release-${{ env.PKG_VERSION }} - path: /home/runner/work/release/release.tar.gz + with: + buildVersion: ${{ needs.prepare.outputs.buildVersion }} + isReleaseBuild: false + previousVersion: ${{ needs.prepare.outputs.previousVersion }} + handleCoverageResults: ${{ inputs.skipTests == 'false' }} + updateCoverageBaseline: ${{ inputs.updateCoverage }} + baseImageVersion: ${{ needs.prepare.outputs.baseImageVersion }} + skipArmBuild: ${{ inputs.skipArmBuild }} # ----------------------------------------------------------------- # NOTIFY @@ -352,10 +133,10 @@ jobs: notify: name: Notify if: ${{ always() }} - needs: [prepare, tests, release] + needs: [prepare, tests, build] runs-on: ubuntu-latest env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - name: Notify on Slack (Success) @@ -366,7 +147,7 @@ jobs: method: chat.postMessage payload: | channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} - text: "Datatracker Build by ${{ github.triggering_actor }}" + text: "Datatracker Dev Build by ${{ github.triggering_actor }}" attachments: - color: "28a745" fields: @@ -381,26 +162,26 @@ jobs: method: chat.postMessage payload: | channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} - text: "Datatracker Build by ${{ github.triggering_actor }}" + text: "Datatracker Dev Build by ${{ github.triggering_actor }}" attachments: - color: "a82929" fields: - title: "Status" short: true - value: "Failed" + value: "Failed 😱" # ----------------------------------------------------------------- # DEV # ----------------------------------------------------------------- dev: name: Deploy to Dev - if: ${{ !failure() && !cancelled() && github.event.inputs.dev == 'true' }} - needs: [prepare, release] + if: ${{ !failure() && !cancelled() && inputs.deploy == 'Dev' }} + needs: [prepare, build] runs-on: ubuntu-latest environment: name: dev env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - uses: actions/checkout@v6 @@ -424,7 +205,7 @@ jobs: repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' + inputs: '{ "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }}, "deployStrategy":"${{ inputs.deployStrategy }}" }' waitForCompletionTimeout: 60m # ----------------------------------------------------------------- @@ -432,13 +213,13 @@ jobs: # ----------------------------------------------------------------- staging: name: Deploy to Staging - if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging Only' || github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }} - needs: [prepare, release] + if: ${{ !failure() && !cancelled() && inputs.deploy == 'Staging' }} + needs: [prepare, build] runs-on: ubuntu-latest environment: name: staging env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - name: Refresh Staging DB @@ -458,29 +239,5 @@ jobs: repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' - waitForCompletionTimeout: 30m - - # ----------------------------------------------------------------- - # PROD - # ----------------------------------------------------------------- - prod: - name: Deploy to Production - if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }} - needs: [prepare, staging] - runs-on: ubuntu-latest - environment: - name: production - env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - - steps: - - name: Deploy to production - uses: ietf-tools/workflow-dispatch-action@v1 - with: - workflow: deploy.yml - repo: ietf-tools/infra-k8s - ref: main - token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' waitForCompletionTimeout: 30m diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml index 5349f1ac7a6..4d4e034ed64 100644 --- a/.github/workflows/ci-run-tests.yml +++ b/.github/workflows/ci-run-tests.yml @@ -38,7 +38,7 @@ jobs: # ----------------------------------------------------------------- tests: name: Run Tests - uses: ./.github/workflows/tests.yml + uses: ./.github/workflows/reusable-tests.yml needs: [prepare] with: ignoreLowerCoverage: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..437ed1baac8 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,209 @@ +name: Make Release and Deploy +run-name: RELEASE ${{ github.run_number }} by @${{ github.actor }} + +on: + workflow_dispatch: + inputs: + deployStrategy: + description: 'Deploy Strategy' + default: 'Recreate' + required: true + type: choice + options: + - Recreate + - RollingUpdate + - Current + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + # ----------------------------------------------------------------- + # PREPARE + # ----------------------------------------------------------------- + prepare: + name: Prepare Release + runs-on: ubuntu-latest + outputs: + buildVersion: ${{ steps.buildvars.outputs.buildVersion }} + previousVersion: ${{ steps.semver.outputs.current }} + baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} + + steps: + - name: Ensure release branch + if: github.ref != 'refs/heads/release' + run: | + echo "This workflow can only run on the 'release' branch (current: ${{ github.ref_name }})" + exit 1 + + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Get Next Version + id: semver + uses: ietf-tools/semver-action@v1 + with: + token: ${{ github.token }} + branch: release + skipInvalidTags: true + patchList: fix, bugfix, perf, refactor, test, tests, chore + + - name: Create Draft Release + uses: ncipollo/release-action@v1.21.0 + with: + prerelease: true + draft: false + commit: ${{ github.sha }} + tag: ${{ steps.semver.outputs.nextStrict }} + name: ${{ steps.semver.outputs.nextStrict }} + body: '*pending*' + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set Build Variables + id: buildvars + run: | + echo "Will set build version to: ${{ steps.semver.outputs.nextStrict }}" + echo "buildVersion=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT + echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME" + + - name: Get Base Image Target Version + id: baseimgversion + run: | + echo "baseImageVersion=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + + # ----------------------------------------------------------------- + # TESTS + # ----------------------------------------------------------------- + tests: + name: Run Tests + uses: ./.github/workflows/reusable-tests.yml + needs: [prepare] + secrets: inherit + with: + ignoreLowerCoverage: false + skipSelenium: true + targetBaseVersion: ${{ needs.prepare.outputs.baseImageVersion }} + + # ----------------------------------------------------------------- + # BUILD RELEASE + # ----------------------------------------------------------------- + build: + name: Build Release Image + uses: ./.github/workflows/reusable-build.yml + if: ${{ !failure() && !cancelled() }} + needs: [tests, prepare] + permissions: + contents: write + packages: write + secrets: inherit + with: + buildVersion: ${{needs.prepare.outputs.buildVersion}} + isReleaseBuild: true + previousVersion: ${{needs.prepare.outputs.previousVersion}} + handleCoverageResults: true + updateCoverageBaseline: true + baseImageVersion: ${{ needs.prepare.outputs.baseImageVersion }} + + # ----------------------------------------------------------------- + # NOTIFY + # ----------------------------------------------------------------- + notify: + name: Notify + if: ${{ always() }} + needs: [prepare, tests, build] + runs-on: ubuntu-latest + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Notify on Slack (Success) + if: ${{ !contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 + with: + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Release Build by ${{ github.triggering_actor }}" + attachments: + - color: "28a745" + fields: + - title: "Status" + short: true + value: "Completed" + - name: Notify on Slack (Failure) + if: ${{ contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 + with: + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Release Build by ${{ github.triggering_actor }}" + attachments: + - color: "a82929" + fields: + - title: "Status" + short: true + value: "Failed 😱" + + # ----------------------------------------------------------------- + # STAGING + # ----------------------------------------------------------------- + staging: + name: Deploy to Staging + if: ${{ !failure() && !cancelled() }} + needs: [prepare, build] + runs-on: ubuntu-latest + environment: + name: staging + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Refresh Staging DB + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy-db.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }' + waitForCompletionTimeout: 120m + + - name: Deploy to staging + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m + + # ----------------------------------------------------------------- + # PROD + # ----------------------------------------------------------------- + prod: + name: Deploy to Production + if: ${{ !failure() && !cancelled() }} + needs: [prepare, staging] + runs-on: ubuntu-latest + environment: + name: production + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Deploy to production + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml new file mode 100644 index 00000000000..93867d3400e --- /dev/null +++ b/.github/workflows/reusable-build.yml @@ -0,0 +1,226 @@ +name: Reusable Build Workflow + +on: + workflow_call: + inputs: + buildVersion: + required: true + type: string + isReleaseBuild: + default: false + required: false + type: boolean + previousVersion: + required: false + type: string + handleCoverageResults: + default: true + required: false + type: boolean + updateCoverageBaseline: + default: false + required: false + type: boolean + baseImageVersion: + default: latest + required: false + type: string + skipArmBuild: + default: false + required: false + type: boolean + +jobs: + # ----------------------------------------------------------------- + # BUILD IMAGE + # ----------------------------------------------------------------- + build: + name: Build Image + runs-on: + group: hperf-8c32r + permissions: + contents: write + packages: write + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Setup Node.js environment + uses: actions/setup-node@v6 + with: + node-version: 18.x + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.x" + + - name: Setup AWS CLI + uses: unfor19/install-aws-cli-action@v1 + with: + version: 2.22.35 + + - name: Download a Coverage Results + if: ${{ inputs.handleCoverageResults }} + uses: actions/download-artifact@v8.0.1 + with: + name: coverage + + - name: Make Build + env: + DEBIAN_FRONTEND: noninteractive + BROWSERSLIST_IGNORE_OLD_DATA: 1 + run: | + echo "BUILD_VERSION: ${{ inputs.buildVersion }}" + echo "GITHUB_SHA: $GITHUB_SHA" + echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" + echo "Running frontend build script..." + echo "Compiling native node packages..." + yarn rebuild + echo "Packaging static assets..." + yarn build --base=https://static.ietf.org/dt/${{ inputs.buildVersion }}/ + yarn legacy:build + echo "Setting version ${{ inputs.buildVersion }}..." + sed -i -r -e "s|^__version__ += '.*'$|__version__ = '${{ inputs.buildVersion }}'|" ietf/__init__.py + sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py + sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py + + - name: Set Production Flags + if: ${{ inputs.isReleaseBuild }} + run: | + echo "Setting production flags in settings.py..." + sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py + + - name: Make Tarball + env: + DEBIAN_FRONTEND: noninteractive + run: | + echo "Build release tarball..." + mkdir -p /home/runner/work/release + tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . + + - name: Collect + Push Statics + env: + DEBIAN_FRONTEND: noninteractive + AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} + AWS_DEFAULT_REGION: auto + AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} + run: | + echo "Collecting statics..." + echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.baseImageVersion }}" + docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.baseImageVersion }} sh dev/build/collectstatics.sh + echo "Pushing statics..." + cd static + aws s3 sync . s3://static/dt/${{ inputs.buildVersion }} --only-show-errors + + - name: Augment dockerignore for docker image build + env: + DEBIAN_FRONTEND: noninteractive + run: | + cat >> .dockerignore <> $GITHUB_ENV + + - name: Build Images + uses: docker/build-push-action@v7 + env: + DOCKER_BUILD_SUMMARY: false + with: + context: . + file: dev/build/Dockerfile + platforms: ${{ inputs.skipArmBuild && 'linux/amd64' || 'linux/amd64,linux/arm64' }} + push: true + tags: | + ghcr.io/ietf-tools/datatracker:${{ inputs.buildVersion }} + ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} + + - name: Update CHANGELOG + id: changelog + uses: Requarks/changelog-action@v1 + if: ${{ inputs.isReleaseBuild }} + with: + token: ${{ github.token }} + fromTag: ${{ inputs.buildVersion }} + toTag: ${{ inputs.previousVersion }} + writeToFile: false + + - name: Download Coverage Results + if: ${{ inputs.handleCoverageResults }} + uses: actions/download-artifact@v8.0.1 + with: + name: coverage + + - name: Prepare Coverage Action + if: ${{ inputs.handleCoverageResults }} + working-directory: ./dev/coverage-action + run: npm install + + - name: Process Coverage Stats + Chart + id: covprocess + uses: ./dev/coverage-action/ + if: ${{ inputs.handleCoverageResults }} + with: + token: ${{ github.token }} + tokenCommon: ${{ secrets.GH_COMMON_TOKEN }} + repoCommon: common + version: ${{ inputs.buildVersion }} + changelog: ${{ steps.changelog.outputs.changes }} + summary: '' + coverageResultsPath: coverage.json + histCoveragePath: historical-coverage.json + + - name: Create Release + uses: ncipollo/release-action@v1.21.0 + if: ${{ inputs.isReleaseBuild }} + with: + allowUpdates: true + makeLatest: true + draft: false + tag: ${{ inputs.buildVersion }} + name: ${{ inputs.buildVersion }} + body: ${{ steps.covprocess.outputs.changelog }} + artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Update Baseline Coverage + uses: ncipollo/release-action@v1.21.0 + if: ${{ inputs.updateCoverageBaseline }} + with: + allowUpdates: true + tag: baseline + omitBodyDuringUpdate: true + omitNameDuringUpdate: true + omitPrereleaseDuringUpdate: true + replacesArtifacts: true + artifacts: "coverage.json" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload Build Artifacts + uses: actions/upload-artifact@v7 + with: + name: release-${{ inputs.buildVersion }} + path: /home/runner/work/release/release.tar.gz diff --git a/.github/workflows/tests.yml b/.github/workflows/reusable-tests.yml similarity index 100% rename from .github/workflows/tests.yml rename to .github/workflows/reusable-tests.yml From 5dd0fc6ecbc2d9337ebd7e8cf5957dd8a0907b90 Mon Sep 17 00:00:00 2001 From: NGPixel Date: Thu, 11 Jun 2026 22:33:52 -0400 Subject: [PATCH 163/188] ci: fix build workflow --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c449b53ce8..4b2bd03f68f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,6 +118,7 @@ jobs: permissions: contents: write packages: write + secrets: inherit with: buildVersion: ${{ needs.prepare.outputs.buildVersion }} isReleaseBuild: false From 70ed39f7c00f58f67d69243cbd16280eeca0cf06 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Fri, 12 Jun 2026 17:03:45 -0400 Subject: [PATCH 164/188] ci: fix build workflow --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4b2bd03f68f..bfc77b2910e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,7 +14,7 @@ on: - Dev - Staging deployStrategy: - description: 'Deploy Strategy' + description: 'Deploy Strategy (Staging Only)' default: 'Recreate' required: true type: choice @@ -23,7 +23,7 @@ on: - RollingUpdate - Current devNoDbRefresh: - description: 'Dev Disable Daily DB Refresh' + description: 'Disable Daily DB Refresh (Dev Only)' default: false required: true type: boolean @@ -206,7 +206,7 @@ jobs: repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }}, "deployStrategy":"${{ inputs.deployStrategy }}" }' + inputs: '{ "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' waitForCompletionTimeout: 60m # ----------------------------------------------------------------- From af603788927fc11746be350b4fbb4d6a15ca27b8 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 16 Jun 2026 09:01:58 +1200 Subject: [PATCH 165/188] fix: scrub control characters from object values headed towards lxml (#11040) * fix: scrub control characters from object values headed towards lxml * chore: clearer comment --- ietf/api/__init__.py | 28 +++++++++++++++++----------- ietf/api/tests.py | 27 +++++++++++++++++---------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index d4562f97ddf..00733717c4f 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -177,8 +177,15 @@ def dehydrate(self, bundle, for_list=True): return dehydrated + +# XML 1.0 forbids all control characters except tab (#x9), LF (#xA), and CR (#xD). +# Replace each with its Unicode control picture (U+2400 + codepoint) so the +# substitution is lossless and the result is valid XML. +_XML_INVALID_CTRL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]") + + class Serializer(tastypie.serializers.Serializer): - OPTION_ESCAPE_NULLS = "datatracker-escape-nulls" + OPTION_ESCAPE_XML_INVALID = "datatracker-escape-xml-invalid" def format_datetime(self, data): return data.astimezone(datetime.UTC).replace(tzinfo=None).isoformat(timespec="seconds") + "Z" @@ -186,18 +193,17 @@ def format_datetime(self, data): def to_simple(self, data, options): options = options or {} simple_data = super().to_simple(data, options) - if ( - options.get(self.OPTION_ESCAPE_NULLS, False) - and isinstance(simple_data, str) - ): - # replace nulls with unicode "symbol for null character", \u2400 - simple_data = simple_data.replace("\x00", "\u2400") + if options.get(self.OPTION_ESCAPE_XML_INVALID, False) and isinstance(simple_data, str): + # Replace control chars invalid in XML 1.0 with their Unicode + # control pictures (U+2400-U+241F) so lxml won't reject the string. + simple_data = _XML_INVALID_CTRL_RE.sub( + lambda m: chr(ord(m.group()) + 0x2400), simple_data + ) return simple_data def to_etree(self, data, options=None, name=None, depth=0): - # lxml does not escape nulls on its own, so ask to_simple() to do it. - # This is mostly (only?) an issue when generating errors responses for - # fuzzers. + # lxml rejects control characters that are invalid in XML 1.0. + # Ask to_simple() to escape them before they reach lxml. options = options or {} - options[self.OPTION_ESCAPE_NULLS] = True + options[self.OPTION_ESCAPE_XML_INVALID] = True return super().to_etree(data, options, name, depth) diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 2a44791a5cf..887969cec15 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -1542,20 +1542,27 @@ def test_all_model_resources_exist(self): self.assertIn(model._meta.model_name, list(app_resources.keys()), "There doesn't seem to be any API resource for model %s.models.%s"%(app.__name__,model.__name__,)) - def test_serializer_to_etree_handles_nulls(self): - """Serializer to_etree() should handle a null character""" + def test_serializer_to_etree_handles_xml_invalid_control_chars(self): + """Serializer to_etree() must not raise ValueError for any XML-invalid control character.""" serializer = Serializer() + # Ordinary strings and strings with valid whitespace must pass through unchanged. try: - serializer.to_etree("string with no nulls in it") + serializer.to_etree("string with no special chars") + serializer.to_etree("tab\there lf\nhere cr\rhere") except ValueError: self.fail("serializer.to_etree raised ValueError on an ordinary string") - try: - serializer.to_etree("string with a \x00 in it") - except ValueError: - self.fail( - "serializer.to_etree raised ValueError on a string " - "containing a null character" - ) + # Every control character that XML 1.0 forbids must be escaped rather than + # causing a ValueError. This is the class of characters that triggered the + # production exception (lxml.etree._utf8 rejects them all). + invalid_chars = [chr(c) for c in list(range(0x00, 0x09)) + [0x0b, 0x0c] + list(range(0x0e, 0x20))] + for ch in invalid_chars: + try: + serializer.to_etree(f"string with {ch!r} in it") + except ValueError: + self.fail( + f"serializer.to_etree raised ValueError on a string " + f"containing control character U+{ord(ch):04X}" + ) class RfcdiffSupportTests(TestCase): From d69e5457544e061177236800b592c6986bb5b0f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jun 2026 10:56:03 +1200 Subject: [PATCH 166/188] chore(deps): bump actions/checkout from 6 to 7 (#11074) Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-base-app.yml | 2 +- .github/workflows/build-devblobstore.yml | 2 +- .github/workflows/build-mq-broker.yml | 2 +- .github/workflows/build.yml | 4 ++-- .github/workflows/ci-run-tests.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/dev-assets-sync-nightly.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/reusable-build.yml | 2 +- .github/workflows/reusable-tests.yml | 6 +++--- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 35172aa2996..a2abe089ce6 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -18,7 +18,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: token: ${{ secrets.GH_COMMON_TOKEN }} diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml index 14c4b1a135a..429aa08b006 100644 --- a/.github/workflows/build-devblobstore.yml +++ b/.github/workflows/build-devblobstore.yml @@ -20,7 +20,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index b297e34b476..7832e65a3a3 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -24,7 +24,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up QEMU uses: docker/setup-qemu-action@v4 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bfc77b2910e..5088d763e92 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,7 +65,7 @@ jobs: baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false @@ -185,7 +185,7 @@ jobs: BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: ref: main diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml index 4d4e034ed64..50ec4c1943b 100644 --- a/.github/workflows/ci-run-tests.yml +++ b/.github/workflows/ci-run-tests.yml @@ -23,7 +23,7 @@ jobs: base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bc20779ae6c..15b2231ce39 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Initialize CodeQL uses: github/codeql-action/init@v4 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index e255b270ff9..7393e83b829 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: 'Dependency Review' uses: actions/dependency-review-action@v4 with: diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index cd986f06f33..83121354e64 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -29,7 +29,7 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Login to GitHub Container Registry uses: docker/login-action@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 437ed1baac8..2de02eb2852 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: echo "This workflow can only run on the 'release' branch (current: ${{ github.ref_name }})" exit 1 - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index 93867d3400e..4d9a3b4bf11 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -43,7 +43,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml index ad2e35408d4..10d04c8265e 100644 --- a/.github/workflows/reusable-tests.yml +++ b/.github/workflows/reusable-tests.yml @@ -32,7 +32,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-devblobstore:latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Prepare for tests run: | @@ -102,7 +102,7 @@ jobs: project: [chromium, firefox] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: actions/setup-node@v6 with: @@ -144,7 +144,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-db:latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Prepare for tests run: | From a89f27bcf5bb7d45119ae2cad5b6b553f91888c9 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 24 Jun 2026 04:19:39 +1200 Subject: [PATCH 167/188] fix: Use non-four digit RFC numbers for DOI (#11080) --- ietf/doc/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 156bac4e779..3685ab65519 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -163,7 +163,7 @@ class DocumentInfo(models.Model): @property def doi(self) -> str | None: if self.type_id == "rfc" and self.rfc_number is not None: - return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number:04d}" + return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number}" return None def file_extension(self): From b05e281a7f7ad15106bd06ad124ef52ee546c6f0 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 1 Jul 2026 07:51:31 +1200 Subject: [PATCH 168/188] fix: include URL to I-D Archive in WGLC messages (#11056) --- ietf/doc/models.py | 5 ++++ ietf/doc/tests_draft.py | 53 ++++++++++++++++++++++++++++++++++++++++- ietf/doc/views_draft.py | 1 + 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 3685ab65519..77b2c86efe5 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -344,6 +344,11 @@ def _get_ref(self, meeting=None, versioned=True): setattr(self, cache_attr, href) return getattr(self, cache_attr) + def refresh_from_db(self, using=None, fields=None, **kwargs): + super().refresh_from_db(using=using, fields=fields, **kwargs) + self.state_cache = None + self._cached_state_slug = {} + def set_state(self, state): """Switch state type implicit in state to state. This just sets the state, doesn't log the change.""" diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index db9dbc2baff..688b933aad3 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -2187,9 +2187,34 @@ def _assert_rejected(testcase, doc, person): chair = RoleFactory(name_id="chair", group=doc.group).person url = urlreverse("ietf.doc.views_draft.issue_wg_lc", kwargs=dict(name=doc.name)) login_testing_unauthorized(self, chair.user.username, url) + + # GET: check 200 and that the pre-populated body contains the archive URL r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) + prefilled_body = q("textarea#id_body").text() + expected_archive_url = f"{settings.IETF_ID_ARCHIVE_URL}{doc.name}-{doc.rev}.txt" + self.assertIn( + expected_archive_url, + prefilled_body, + "Pre-populated body must contain the I-D archive .txt URL", + ) + + # Invalid POST: end_date in the past must not change state + bad_postdict = dict( + end_date=date_today(DEADLINE_TZINFO), + to=q("input#id_to").attr("value"), + subject=q("input#id_subject").attr("value"), + body=prefilled_body, + ) + r = self.client.post(url, bad_postdict) + self.assertEqual(r.status_code, 200, "Invalid POST should re-render the form") + self.assertNotEqual( + doc.get_state_slug("draft-stream-ietf"), + "wg-lc", + "State must not change on invalid POST", + ) + postdict = dict() postdict["end_date"] = q("input#id_end_date").attr("value") postdict["to"] = q("input#id_to").attr("value") + ", extrato@example.org" @@ -2199,15 +2224,36 @@ def _assert_rejected(testcase, doc, person): else: postdict["cc"] = "extracc@example.org" postdict["subject"] = q("input#id_subject").attr("value") + " Extra Subject Words" - postdict["body"] = q("textarea#id_body").text() + "FGgqbQ$UNeXs" + postdict["body"] = prefilled_body + "FGgqbQ$UNeXs" empty_outbox() r = self.client.post( url, postdict, ) self.assertEqual(r.status_code, 302) + doc.refresh_from_db() self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "wg-lc") + + # A StateDocEvent recording the transition must exist + self.assertTrue( + doc.docevent_set.filter(type="changed_state").exists(), + "Expected a changed_state event after issuing WG LC", + ) + + # A stream-s reminder must be active for the doc + self.assertTrue( + DocReminder.objects.filter( + event__doc=doc, type_id="stream-s", active=True + ).exists(), + "Expected an active stream-s reminder after issuing WG LC", + ) + self.assertEqual(len(outbox), 2) + + # outbox[0] is the stream-state-changed notification + self.assertIn(doc.name, outbox[0]["Subject"]) + + # outbox[1] is the WGLC announcement sent to the WG list self.assertIn(f"{doc.group.acronym}@ietf.org", outbox[1]["To"]) self.assertIn("extrato@example.org", outbox[1]["To"]) self.assertIn("extracc@example.org", outbox[1]["Cc"]) @@ -2216,6 +2262,11 @@ def _assert_rejected(testcase, doc, person): body = get_payload_text(outbox[1]) self.assertIn("disclosure obligations", body) self.assertIn("FGgqbQ$UNeXs", body) + self.assertIn( + expected_archive_url, + body, + "WGLC email body must contain the I-D archive .txt URL", + ) def test_issue_wg_call_for_adoption_form(self): end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=1) diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py index 0f5ea49f5d4..a64d0a53fe9 100644 --- a/ietf/doc/views_draft.py +++ b/ietf/doc/views_draft.py @@ -1973,6 +1973,7 @@ def issue_wg_lc(request, name): doc=doc, end_date=end_date, wg_list=doc.group.list_email, + url=f"{settings.IETF_ID_ARCHIVE_URL}{doc.name}-{doc.rev}.txt", settings=settings, ), ) From f158244c292d725487c7e810de8b174bd46f5765 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 1 Jul 2026 07:55:57 +1200 Subject: [PATCH 169/188] fix: remove unreachable old RFC-Editor sync code. Flatten redirects. (#10967) --- ietf/doc/models.py | 3 - ietf/settings.py | 1 - ietf/sync/rfceditor.py | 845 ------------------ ietf/sync/tasks.py | 89 -- ietf/sync/tests.py | 687 +------------- ietf/sync/views.py | 19 +- ietf/templates/doc/document_draft.html | 2 +- ietf/templates/doc/search/status_columns.html | 2 +- .../management/commands/periodic_tasks.py | 28 - ietf/utils/timezone.py | 2 +- 10 files changed, 6 insertions(+), 1672 deletions(-) delete mode 100644 ietf/sync/rfceditor.py diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 77b2c86efe5..ff879dd71c6 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1283,9 +1283,6 @@ def submission(self): def pub_datetime(self): """Get the publication datetime of this document""" if self.type_id == "rfc": - # As of Sept 2022, in ietf.sync.rfceditor.update_docs_from_rfc_index() `published_rfc` events are - # created with a timestamp whose date *in the PST8PDT timezone* is the official publication date - # assigned by the RFC editor. event = self.latest_event(type='published_rfc') else: event = self.latest_event(type='new_revision') diff --git a/ietf/settings.py b/ietf/settings.py index d509a877c61..7af9c7ae5be 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -895,7 +895,6 @@ def skip_unreadable_post(record): IANA_SYNC_PROTOCOLS_URL = "https://www.iana.org/protocols/" RFC_EDITOR_GROUP_NOTIFICATION_EMAIL = "webmaster@rfc-editor.org" -RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" RFC_EDITOR_ERRATA_JSON_URL = "https://www.rfc-editor.org/errata.json" RFC_EDITOR_INLINE_ERRATA_URL = "https://www.rfc-editor.org/rfc/inline-errata/rfc{rfc_number}.html" diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py deleted file mode 100644 index 347b58efbb2..00000000000 --- a/ietf/sync/rfceditor.py +++ /dev/null @@ -1,845 +0,0 @@ -# Copyright The IETF Trust 2012-2025, All Rights Reserved -# -*- coding: utf-8 -*- - - -import datetime -import re - -from typing import Iterator, Optional, Union -from xml.dom import pulldom, Node - -from django.db import transaction -from django.db.models import Subquery, OuterRef, F, Q -from django.utils import timezone - -import debug # pyflakes:ignore - -from ietf.doc.models import ( Document, State, StateType, DocEvent, DocRelationshipName, - DocTagName, RelatedDocument, RelatedDocHistory ) -from ietf.doc.expire import move_draft_files_to_archive -from ietf.doc.utils import add_state_change_event, new_state_change_event, prettify_std_name, update_action_holders -from ietf.group.models import Group -from ietf.ipr.models import IprDocRel -from ietf.name.models import StdLevelName, StreamName -from ietf.person.models import Person -from ietf.utils.log import log -from ietf.utils.mail import send_mail_text -from ietf.utils.timezone import datetime_from_date, RPC_TZINFO - -# QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" -# INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" -# POST_APPROVED_DRAFT_URL = "https://www.rfc-editor.org/sdev/jsonexp/jsonparser.php" - -MIN_ERRATA_RESULTS = 5000 -MIN_INDEX_RESULTS = 8000 -MIN_QUEUE_RESULTS = 10 - -def get_child_text(parent_node, tag_name): - text = [] - for node in parent_node.childNodes: - if node.nodeType == Node.ELEMENT_NODE and node.localName == tag_name: - text.append(node.firstChild.data) - return '\n\n'.join(text) - - -def parse_queue(response): - """Parse RFC Editor queue XML into a bunch of tuples + warnings.""" - - events = pulldom.parse(response) - drafts = [] - warnings = [] - stream = None - - for event, node in events: - try: - if event == pulldom.START_ELEMENT and node.tagName == "entry": - events.expandNode(node) - node.normalize() - draft_name = get_child_text(node, "draft").strip() - draft_name = re.sub(r"(-\d\d)?(.txt){1,2}$", "", draft_name) - date_received = get_child_text(node, "date-received") - - state = "" - tags = [] - missref_generation = "" - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "state": - state = child.firstChild.data - # state has some extra annotations encoded, parse - # them out - if '*R' in state: - tags.append("ref") - state = state.replace("*R", "") - if '*A' in state: - tags.append("iana") - state = state.replace("*A", "") - m = re.search(r"\(([0-9]+)G\)", state) - if m: - missref_generation = m.group(1) - state = state.replace("(%sG)" % missref_generation, "") - - # AUTH48 link - auth48 = "" - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "auth48-url": - auth48 = child.firstChild.data - - # cluster link (if it ever gets implemented) - cluster = "" - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "cluster-url": - cluster = child.firstChild.data - - refs = [] - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "normRef": - ref_name = get_child_text(child, "ref-name") - ref_state = get_child_text(child, "ref-state") - in_queue = ref_state.startswith("IN-QUEUE") - refs.append((ref_name, ref_state, in_queue)) - - drafts.append((draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs)) - - elif event == pulldom.START_ELEMENT and node.tagName == "section": - name = node.getAttribute('name') - if name.startswith("IETF"): - stream = "ietf" - elif name.startswith("IAB"): - stream = "iab" - elif name.startswith("IRTF"): - stream = "irtf" - elif name.startswith("INDEPENDENT"): - stream = "ise" - elif name.startswith("Editorial Stream"): - stream = "editorial" - else: - stream = None - warnings.append("unrecognized section " + name) - except Exception as e: - log("Exception when processing an RFC queue entry: %s" % e) - log("node: %s" % node) - raise - - return drafts, warnings - -def update_drafts_from_queue(drafts): - """Given a list of parsed drafts from the RFC Editor queue, update the - documents in the database. Return those that were changed.""" - - tag_mapping = { - 'IANA': DocTagName.objects.get(slug='iana'), - 'REF': DocTagName.objects.get(slug='ref') - } - - slookup = dict((s.slug, s) - for s in State.objects.filter(used=True, type=StateType.objects.get(slug="draft-rfceditor"))) - state_mapping = { - 'AUTH': slookup['auth'], - 'AUTH48': slookup['auth48'], - 'AUTH48-DONE': slookup['auth48-done'], - 'EDIT': slookup['edit'], - 'IANA': slookup['iana'], - 'IESG': slookup['iesg'], - 'ISR': slookup['isr'], - 'ISR-AUTH': slookup['isr-auth'], - 'REF': slookup['ref'], - 'RFC-EDITOR': slookup['rfc-edit'], - 'TI': slookup['tooling-issue'], - 'TO': slookup['timeout'], - 'MISSREF': slookup['missref'], - } - - system = Person.objects.get(name="(System)") - - warnings = [] - - names = [t[0] for t in drafts] - - drafts_in_db = dict((d.name, d) - for d in Document.objects.filter(type="draft", name__in=names)) - - changed = set() - - for name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs in drafts: - if name not in drafts_in_db: - warnings.append("unknown document %s" % name) - continue - - if not state or state not in state_mapping: - warnings.append("unknown state '%s' for %s" % (state, name)) - continue - - d = drafts_in_db[name] - - prev_state = d.get_state("draft-rfceditor") - next_state = state_mapping[state] - events = [] - - # check if we've noted it's been received - if d.get_state_slug("draft-iesg") == "ann" and not prev_state and not d.latest_event(DocEvent, type="rfc_editor_received_announcement"): - e = DocEvent(doc=d, rev=d.rev, by=system, type="rfc_editor_received_announcement") - e.desc = "Announcement was received by RFC Editor" - e.save() - send_mail_text(None, "iesg-secretary@ietf.org", None, - '%s in RFC Editor queue' % d.name, - 'The announcement for %s has been received by the RFC Editor.' % d.name) - # change draft-iesg state to RFC Ed Queue - prev_iesg_state = State.objects.get(used=True, type="draft-iesg", slug="ann") - next_iesg_state = State.objects.get(used=True, type="draft-iesg", slug="rfcqueue") - - d.set_state(next_iesg_state) - e = add_state_change_event(d, system, prev_iesg_state, next_iesg_state) - if e: - events.append(e) - e = update_action_holders(d, prev_iesg_state, next_iesg_state) - if e: - events.append(e) - changed.add(name) - - # check draft-rfceditor state - if prev_state != next_state: - d.set_state(next_state) - - e = new_state_change_event(d, system, prev_state, next_state) # unsaved - if e: - if auth48: - e.desc = re.sub(r"(.*)", "\\1" % auth48, e.desc) - e.save() - events.append(e) - - if auth48: - # Create or update the auth48 URL whether or not this is a state expected to have one. - d.documenturl_set.update_or_create( - tag_id='auth48', # look up existing based on this field - defaults=dict(url=auth48) # create or update with this field - ) - else: - # Remove any existing auth48 URL when an update does not have one. - d.documenturl_set.filter(tag_id='auth48').delete() - - changed.add(name) - - t = DocTagName.objects.filter(slug__in=tags) - if set(t) != set(d.tags.all()): - d.tags.clear() - d.tags.set(t) - changed.add(name) - - if events: - d.save_with_history(events) - - - # remove tags and states for those not in the queue anymore - for d in Document.objects.exclude(name__in=names).filter(states__type="draft-rfceditor").distinct(): - d.tags.remove(*list(tag_mapping.values())) - d.unset_state("draft-rfceditor") - # we do not add a history entry here - most likely we already - # have something that explains what happened - changed.add(name) - - return changed, warnings - - -def parse_index(response): - """Parse RFC Editor index XML into a bunch of tuples.""" - - def normalize_std_name(std_name): - # remove zero padding - prefix = std_name[:3] - if prefix in ("RFC", "FYI", "BCP", "STD"): - try: - return prefix + str(int(std_name[3:])) - except ValueError: - pass - return std_name - - def extract_doc_list(parentNode, tagName): - l = [] - for u in parentNode.getElementsByTagName(tagName): - for d in u.getElementsByTagName("doc-id"): - l.append(normalize_std_name(d.firstChild.data)) - return l - - also_list = {} - data = [] - events = pulldom.parse(response) - for event, node in events: - try: - if event == pulldom.START_ELEMENT and node.tagName in ["bcp-entry", "fyi-entry", "std-entry"]: - events.expandNode(node) - node.normalize() - bcpid = normalize_std_name(get_child_text(node, "doc-id")) - doclist = extract_doc_list(node, "is-also") - for docid in doclist: - if docid in also_list: - also_list[docid].append(bcpid) - else: - also_list[docid] = [bcpid] - - elif event == pulldom.START_ELEMENT and node.tagName == "rfc-entry": - events.expandNode(node) - node.normalize() - rfc_number = int(get_child_text(node, "doc-id")[3:]) - title = get_child_text(node, "title") - - authors = [] - for author in node.getElementsByTagName("author"): - authors.append(get_child_text(author, "name")) - - d = node.getElementsByTagName("date")[0] - year = int(get_child_text(d, "year")) - month = get_child_text(d, "month") - month = ["January","February","March","April","May","June","July","August","September","October","November","December"].index(month)+1 - rfc_published_date = datetime.date(year, month, 1) - - current_status = get_child_text(node, "current-status").title() - - updates = extract_doc_list(node, "updates") - updated_by = extract_doc_list(node, "updated-by") - obsoletes = extract_doc_list(node, "obsoletes") - obsoleted_by = extract_doc_list(node, "obsoleted-by") - pages = get_child_text(node, "page-count") - stream = get_child_text(node, "stream") - wg = get_child_text(node, "wg_acronym") - if wg and ((wg == "NON WORKING GROUP") or len(wg) > 15): - wg = None - - l = [] - for fmt in node.getElementsByTagName("format"): - l.append(get_child_text(fmt, "file-format")) - file_formats = (",".join(l)).lower() - - abstract = "" - for abstract in node.getElementsByTagName("abstract"): - abstract = get_child_text(abstract, "p") - - draft = get_child_text(node, "draft") - if draft and re.search(r"-\d\d$", draft): - draft = draft[0:-3] - - if len(node.getElementsByTagName("errata-url")) > 0: - has_errata = 1 - else: - has_errata = 0 - - data.append((rfc_number,title,authors,rfc_published_date,current_status,updates,updated_by,obsoletes,obsoleted_by,[],draft,has_errata,stream,wg,file_formats,pages,abstract)) - except Exception as e: - log("Exception when processing an RFC index entry: %s" % e) - log("node: %s" % node) - raise - for d in data: - k = "RFC%d" % d[0] - if k in also_list: - d[9].extend(also_list[k]) - return data - - -def update_docs_from_rfc_index( - index_data, errata_data, skip_older_than_date: Optional[datetime.date] = None -) -> Iterator[tuple[int, list[str], Document, bool]]: - """Given parsed data from the RFC Editor index, update the documents in the database - - Returns an iterator that yields (rfc_number, change_list, doc, rfc_published) for the - RFC document and, if applicable, the I-D that it came from. - - The skip_older_than_date is a bare date, not a datetime. - """ - # Create dict mapping doc-id to list of errata records that apply to it - errata: dict[str, list[dict]] = {} - for item in errata_data: - name = item["doc-id"] - if name.upper().startswith("RFC"): - name = f"RFC{int(name[3:])}" # removes leading 0s on the rfc number - if not name in errata: - errata[name] = [] - errata[name].append(item) - - std_level_mapping = { - "Standard": StdLevelName.objects.get(slug="std"), - "Internet Standard": StdLevelName.objects.get(slug="std"), - "Draft Standard": StdLevelName.objects.get(slug="ds"), - "Proposed Standard": StdLevelName.objects.get(slug="ps"), - "Informational": StdLevelName.objects.get(slug="inf"), - "Experimental": StdLevelName.objects.get(slug="exp"), - "Best Current Practice": StdLevelName.objects.get(slug="bcp"), - "Historic": StdLevelName.objects.get(slug="hist"), - "Unknown": StdLevelName.objects.get(slug="unkn"), - } - - stream_mapping = { - "IETF": StreamName.objects.get(slug="ietf"), - "INDEPENDENT": StreamName.objects.get(slug="ise"), - "IRTF": StreamName.objects.get(slug="irtf"), - "IAB": StreamName.objects.get(slug="iab"), - "Editorial": StreamName.objects.get(slug="editorial"), - "Legacy": StreamName.objects.get(slug="legacy"), - } - - tag_has_errata = DocTagName.objects.get(slug="errata") - tag_has_verified_errata = DocTagName.objects.get(slug="verified-errata") - relationship_obsoletes = DocRelationshipName.objects.get(slug="obs") - relationship_updates = DocRelationshipName.objects.get(slug="updates") - rfc_published_state = State.objects.get(type_id="rfc", slug="published") - - system = Person.objects.get(name="(System)") - - first_sync_creating_subseries = not Document.objects.filter(type_id__in=["bcp","std","fyi"]).exists() - - for ( - rfc_number, - title, - authors, - rfc_published_date, - current_status, - updates, - updated_by, - obsoletes, - obsoleted_by, - also, - draft_name, - has_errata, - stream, - wg, - file_formats, - pages, - abstract, - ) in index_data: - if skip_older_than_date and rfc_published_date < skip_older_than_date: - # speed up the process by skipping old entries (n.b., the comparison above is a - # lexical comparison between "YYYY-MM-DD"-formatted dates) - continue - - # we assume two things can happen: we get a new RFC, or an - # attribute has been updated at the RFC Editor (RFC Editor - # attributes take precedence over our local attributes) - rfc_events = [] - rfc_changes = [] - rfc_published = False - - # Find the draft, if any - draft = None - if draft_name: - try: - draft = Document.objects.get(name=draft_name, type_id="draft") - except Document.DoesNotExist: - pass - # Logging below warning turns out to be unhelpful - there are many references - # to such things in the index: - # * all april-1 RFCs have an internal name that looks like a draft name, but there - # was never such a draft. More of these will exist in the future - # * Several documents were created with out-of-band input to the RFC-editor, for a - # variety of reasons. - # - # What this exposes is that the rfc-index needs to stop talking about these things. - # If there is no draft to point to, don't point to one, even if there was an RPC - # internal name in use (and in the RPC database). This will be a requirement on the - # reimplementation of the creation of the rfc-index. - # - # log(f"Warning: RFC index for {rfc_number} referred to unknown draft {draft_name}") - - # Find or create the RFC document - creation_args: dict[str, Optional[Union[str, int]]] = {"name": f"rfc{rfc_number}"} - if draft: - creation_args.update( - { - "title": draft.title, - "stream": draft.stream, - "group": draft.group, - "abstract": draft.abstract, - "pages": draft.pages, - "words": draft.words, - "std_level": draft.std_level, - "ad": draft.ad, - "external_url": draft.external_url, - "uploaded_filename": draft.uploaded_filename, - "note": draft.note, - } - ) - doc, created_rfc = Document.objects.get_or_create( - rfc_number=rfc_number, type_id="rfc", defaults=creation_args - ) - if created_rfc: - rfc_changes.append(f"created document {prettify_std_name(doc.name)}") - doc.set_state(rfc_published_state) - if draft: - doc.formal_languages.set(draft.formal_languages.all()) - # Create authors based on the last draft in the datatracker. This - # path will go away when we publish via the modernized RPC workflow - # but until then, these are the only data we have for authors that - # are easily connected to Person records. - for documentauthor in draft.documentauthor_set.all(): - # Copy the author but point at the new doc. - # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances - documentauthor.pk = None - documentauthor.id = None - documentauthor._state.adding = True - documentauthor.document = doc - documentauthor.save() - - if draft: - draft_events = [] - draft_changes = [] - - # Ensure the draft is in the "rfc" state and move its files to the archive - # if necessary. - if draft.get_state_slug() != "rfc": - draft.set_state( - State.objects.get(used=True, type="draft", slug="rfc") - ) - move_draft_files_to_archive(draft, draft.rev) - draft_changes.append(f"changed state to {draft.get_state()}") - - # Ensure the draft and rfc are linked with a "became_rfc" relationship - r, created_relateddoc = RelatedDocument.objects.get_or_create( - source=draft, target=doc, relationship_id="became_rfc" - ) - if created_relateddoc: - change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format( - rel_name=r.relationship.name.lower(), - pretty_draft_name=prettify_std_name(draft_name), - pretty_rfc_name=prettify_std_name(doc.name), - ) - draft_changes.append(change) - rfc_changes.append(change) - - # Always set the "draft-iesg" state. This state should be set for all drafts, so - # log a warning if it is not set. What should happen here is that ietf stream - # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index. - # Other stream documents should normally be "idexists" and be left that way. The - # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub", - # and changes any other state to "pub". If unset, it changes it to "idexists". - # This reflects historical behavior and should probably be updated, but a migration - # of existing drafts (and validation of the change) is needed before we change the - # handling. - prev_iesg_state = draft.get_state("draft-iesg") - if prev_iesg_state is None: - log(f'Warning while processing {doc.name}: {draft.name} has no "draft-iesg" state') - new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists") - elif prev_iesg_state.slug not in ("pub", "idexists"): - if prev_iesg_state.slug != "rfcqueue": - log( - 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format( - doc.name, draft.name, prev_iesg_state.slug - ) - ) - new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub") - else: - new_iesg_state = prev_iesg_state - - if new_iesg_state != prev_iesg_state: - draft.set_state(new_iesg_state) - draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}") - e = update_action_holders(draft, prev_iesg_state, new_iesg_state) - if e: - draft_events.append(e) - - # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain. - if draft.stream != doc.stream: - log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format( - doc.name, draft.name, draft.stream, doc.stream - )) - elif draft.stream.slug in ["iab", "irtf", "ise"]: - stream_slug = f"draft-stream-{draft.stream.slug}" - prev_state = draft.get_state(stream_slug) - if prev_state is not None and prev_state.slug != "pub": - new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub") - draft.set_state(new_state) - draft_changes.append( - f"changed {new_state.type.label} to {new_state}" - ) - e = update_action_holders(draft, prev_state, new_state) - if e: - draft_events.append(e) - if draft_changes: - draft_events.append( - DocEvent.objects.create( - doc=draft, - rev=doc.rev, - by=system, - type="sync_from_rfc_editor", - desc=f"Received changes through RFC Editor sync ({', '.join(draft_changes)})", - ) - ) - draft.save_with_history(draft_events) - yield rfc_number, draft_changes, draft, False # yield changes to the draft - - # check attributes - verbed = "set" if created_rfc else "changed" - if title != doc.title: - doc.title = title - rfc_changes.append(f"{verbed} title to '{doc.title}'") - - if abstract and abstract != doc.abstract: - doc.abstract = abstract - rfc_changes.append(f"{verbed} abstract to '{doc.abstract}'") - - if pages and int(pages) != doc.pages: - doc.pages = int(pages) - rfc_changes.append(f"{verbed} pages to {doc.pages}") - - if std_level_mapping[current_status] != doc.std_level: - doc.std_level = std_level_mapping[current_status] - rfc_changes.append(f"{verbed} standardization level to {doc.std_level}") - - if doc.stream != stream_mapping[stream]: - doc.stream = stream_mapping[stream] - rfc_changes.append(f"{verbed} stream to {doc.stream}") - - if doc.get_state() != rfc_published_state: - doc.set_state(rfc_published_state) - rfc_changes.append(f"{verbed} {rfc_published_state.type.label} to {rfc_published_state}") - - # if we have no group assigned, check if RFC Editor has a suggestion - if not doc.group: - if wg: - doc.group = Group.objects.get(acronym=wg) - rfc_changes.append(f"set group to {doc.group}") - else: - doc.group = Group.objects.get( - type="individ" - ) # fallback for newly created doc - - if not doc.latest_event(type="published_rfc"): - e = DocEvent(doc=doc, rev=doc.rev, type="published_rfc") - # unfortunately, rfc_published_date doesn't include the correct day - # at the moment because the data only has month/year, so - # try to deduce it - # - # Note: This is in done PST8PDT to preserve compatibility with events created when - # USE_TZ was False. The published_rfc event was created with a timestamp whose - # server-local datetime (PST8PDT) matched the publication date from the RFC index. - # When switching to USE_TZ=True, the timestamps were migrated so they still - # matched the publication date in PST8PDT. When interpreting the event timestamp - # as a publication date, you must treat it in the PST8PDT time zone. The - # RPC_TZINFO constant in ietf.utils.timezone is defined for this purpose. - d = datetime_from_date(rfc_published_date, RPC_TZINFO) - synthesized = timezone.now().astimezone(RPC_TZINFO) - if abs(d - synthesized) > datetime.timedelta(days=60): - synthesized = d - else: - direction = -1 if (d - synthesized).total_seconds() < 0 else +1 - while synthesized.month != d.month or synthesized.year != d.year: - synthesized += datetime.timedelta(days=direction) - e.time = synthesized - e.by = system - e.desc = "RFC published" - e.save() - rfc_events.append(e) - - rfc_changes.append( - f"added RFC published event at {e.time.strftime('%Y-%m-%d')}" - ) - rfc_published = True - - def parse_relation_list(rel_list: list[str]) -> list[Document]: - return list( - Document.objects.filter( - name__in=[name.strip().lower() for name in rel_list], - type_id="rfc" - ) - ) - - # Create missing obsoletes relations - docs_this_obsoletes = parse_relation_list(obsoletes) - for obs_doc in docs_this_obsoletes: - if not doc.relateddocument_set.filter( - target=obs_doc, relationship=relationship_obsoletes - ): - r = doc.relateddocument_set.create( - target=obs_doc, relationship=relationship_obsoletes - ) - rfc_changes.append( - "created {rel_name} relation between {src} and {tgt}".format( - rel_name=r.relationship.name.lower(), - src=prettify_std_name(r.source.name), - tgt=prettify_std_name(r.target.name), - ) - ) - # Remove stale obsoletes relations - for r in doc.relateddocument_set.filter( - relationship=relationship_obsoletes - ).exclude(target_id__in=[d.pk for d in docs_this_obsoletes]): - r.delete() - rfc_changes.append( - "removed {rel_name} relation between {src} and {tgt}".format( - rel_name=r.relationship.name.lower(), - src=prettify_std_name(r.source.name), - tgt=prettify_std_name(r.target.name), - ) - ) - - docs_this_updates = parse_relation_list(updates) - for upd_doc in docs_this_updates: - if not RelatedDocument.objects.filter( - source=doc, target=upd_doc, relationship=relationship_updates - ): - r = doc.relateddocument_set.create( - target=upd_doc, relationship=relationship_updates - ) - rfc_changes.append( - "created {rel_name} relation between {src} and {tgt}".format( - rel_name=r.relationship.name.lower(), - src=prettify_std_name(r.source.name), - tgt=prettify_std_name(r.target.name), - ) - ) - # Remove stale updates relations - for r in doc.relateddocument_set.filter( - relationship=relationship_updates - ).exclude(target_id__in=[d.pk for d in docs_this_updates]): - r.delete() - rfc_changes.append( - "removed {rel_name} relation between {src} and {tgt}".format( - rel_name=r.relationship.name.lower(), - src=prettify_std_name(r.source.name), - tgt=prettify_std_name(r.target.name), - ) - ) - - if also: - # recondition also to have proper subseries document names: - conditioned_also = [] - for a in also: - a = a.lower() - subseries_slug = a[:3] - if subseries_slug not in ["bcp", "std", "fyi"]: - log(f"Unexpected 'also' relationship of {a} encountered for {doc}") - continue - maybe_number = a[3:].strip() - if not maybe_number.isdigit(): - log(f"Unexpected 'also' subseries element identifier {a} encountered for {doc}") - continue - else: - subseries_number = int(maybe_number) - conditioned_also.append(f"{subseries_slug}{subseries_number}") # Note the lack of leading zeros - also = conditioned_also - - for a in also: - subseries_doc_name = a - subseries_slug=a[:3] - # Leaving most things to the default intentionally - # Of note, title and stream are left to the defaults of "" and none. - subseries_doc, created = Document.objects.get_or_create(type_id=subseries_slug, name=subseries_doc_name) - if created: - if first_sync_creating_subseries: - subseries_doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of this {subseries_slug.upper()} document is currently available in the datatracker before this point") - subseries_doc.docevent_set.create(type=f"{subseries_slug}_doc_created", by=system, desc=f"Imported {subseries_doc_name} into the datatracker via sync to the rfc-index") - else: - subseries_doc.docevent_set.create(type=f"{subseries_slug}_doc_created", by=system, desc=f"Created {subseries_doc_name} via sync to the rfc-index") - _, relationship_created = subseries_doc.relateddocument_set.get_or_create(relationship_id="contains", target=doc) - if relationship_created: - if first_sync_creating_subseries: - subseries_doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Imported membership of {doc.name} in {subseries_doc.name} via sync to the rfc-index") - rfc_events.append(doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of {subseries_doc.name.upper()} is currently available in the datatracker before this point")) - rfc_events.append(doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Imported membership of {doc.name} in {subseries_doc.name} via sync to the rfc-index")) - else: - subseries_doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Added {doc.name} to {subseries_doc.name}") - rfc_events.append(doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Added {doc.name} to {subseries_doc.name}")) - - # Delete subseries relations that are no longer current. Use a transaction - # so we are sure we iterate over the same relations that we delete! - with transaction.atomic(): - stale_subseries_relations = doc.relations_that("contains").exclude( - source__name__in=also - ) - for stale_relation in stale_subseries_relations: - stale_subseries_doc = stale_relation.source - rfc_events.append( - doc.docevent_set.create( - type="sync_from_rfc_editor", - by=system, - desc=f"Removed {doc.name} from {stale_subseries_doc.name}", - ) - ) - stale_subseries_doc.docevent_set.create( - type="sync_from_rfc_editor", - by=system, - desc=f"Removed {doc.name} from {stale_subseries_doc.name}", - ) - stale_subseries_relations.delete() - - doc_errata = errata.get(f"RFC{rfc_number}", []) - all_rejected = doc_errata and all( - er["errata_status_code"] == "Rejected" for er in doc_errata - ) - if has_errata and not all_rejected: - if not doc.tags.filter(pk=tag_has_errata.pk).exists(): - doc.tags.add(tag_has_errata) - rfc_changes.append("added Errata tag") - has_verified_errata = any( - [er["errata_status_code"] == "Verified" for er in doc_errata] - ) - if ( - has_verified_errata - and not doc.tags.filter(pk=tag_has_verified_errata.pk).exists() - ): - doc.tags.add(tag_has_verified_errata) - rfc_changes.append("added Verified Errata tag") - else: - if doc.tags.filter(pk=tag_has_errata.pk): - doc.tags.remove(tag_has_errata) - if all_rejected: - rfc_changes.append("removed Errata tag (all errata rejected)") - else: - rfc_changes.append("removed Errata tag") - if doc.tags.filter(pk=tag_has_verified_errata.pk): - doc.tags.remove(tag_has_verified_errata) - rfc_changes.append("removed Verified Errata tag") - - if rfc_changes: - rfc_events.append( - DocEvent.objects.create( - doc=doc, - rev=doc.rev, - by=system, - type="sync_from_rfc_editor", - desc=f"Received changes through RFC Editor sync ({', '.join(rfc_changes)})", - ) - ) - doc.save_with_history(rfc_events) - yield rfc_number, rfc_changes, doc, rfc_published # yield changes to the RFC - - if first_sync_creating_subseries: - # First - create the known subseries documents that have ghosted. - # The RFC editor (as of 31 Oct 2023) claims these subseries docs do not exist. - # The datatracker, on the other hand, will say that the series doc currently contains no RFCs. - for name in ["fyi17", "std1", "bcp12", "bcp113", "bcp66"]: - # Leaving most things to the default intentionally - # Of note, title and stream are left to the defaults of "" and none. - subseries_doc, created = Document.objects.get_or_create(type_id=name[:3], name=name) - if not created: - log(f"Warning: {name} unexpectedly already exists") - else: - subseries_slug = name[:3] - subseries_doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of this {subseries_slug.upper()} document is currently available in the datatracker before this point") - - RelatedDocument.objects.filter( - Q(originaltargetaliasname__startswith="bcp") | - Q(originaltargetaliasname__startswith="std") | - Q(originaltargetaliasname__startswith="fyi") - ).annotate( - subseries_target=Subquery( - Document.objects.filter(name=OuterRef("originaltargetaliasname")).values_list("pk",flat=True)[:1] - ) - ).update(target=F("subseries_target")) - RelatedDocHistory.objects.filter( - Q(originaltargetaliasname__startswith="bcp") | - Q(originaltargetaliasname__startswith="std") | - Q(originaltargetaliasname__startswith="fyi") - ).annotate( - subseries_target=Subquery( - Document.objects.filter(name=OuterRef("originaltargetaliasname")).values_list("pk",flat=True)[:1] - ) - ).update(target=F("subseries_target")) - IprDocRel.objects.filter( - Q(originaldocumentaliasname__startswith="bcp") | - Q(originaldocumentaliasname__startswith="std") | - Q(originaldocumentaliasname__startswith="fyi") - ).annotate( - subseries_target=Subquery( - Document.objects.filter(name=OuterRef("originaldocumentaliasname")).values_list("pk",flat=True)[:1] - ) - ).update(document=F("subseries_target")) - - - diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index 4ccd5db4bb9..c61e754ed30 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -3,7 +3,6 @@ # Celery task definitions # import datetime -import io from pathlib import Path from tempfile import NamedTemporaryFile import requests @@ -19,13 +18,11 @@ from ietf.person.models import Person from ietf.utils.mail import send_mail_text from ietf.sync import iana -from ietf.sync import rfceditor from ietf.sync.errata import ( errata_are_dirty, mark_errata_as_processed, update_errata_from_rfceditor, ) -from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue from ietf.sync.rfcindex import ( create_bcp_txt_index, create_fyi_txt_index, @@ -36,92 +33,6 @@ ) from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper from ietf.utils import log -from ietf.utils.timezone import date_today - - -@shared_task -def rfc_editor_index_update_task(full_index=False): - """Update metadata from the RFC index - - Default is to examine only changes in the past 365 days. Call with full_index=True to update - the full RFC index. - - According to comments on the original script, a year's worth took about 20s on production as of - August 2022 - - The original rfc-editor-index-update script had a long-disabled provision for running the - rebuild_reference_relations scripts after the update. That has not been brought over - at all because it should be implemented as its own task if it is needed. - """ - skip_date = None if full_index else date_today() - datetime.timedelta(days=365) - log.log( - "Updating document metadata from RFC index going back to {since}, from {url}".format( - since=skip_date if skip_date is not None else "the beginning", - url=settings.RFC_EDITOR_INDEX_URL, - ) - ) - try: - response = requests.get( - settings.RFC_EDITOR_INDEX_URL, - timeout=30, # seconds - ) - except requests.Timeout as exc: - log.log(f"GET request timed out retrieving RFC editor index: {exc}") - return # failed - rfc_index_xml = response.text - index_data = rfceditor.parse_index(io.StringIO(rfc_index_xml)) - try: - response = requests.get( - settings.RFC_EDITOR_ERRATA_JSON_URL, - timeout=30, # seconds - ) - except requests.Timeout as exc: - log.log(f"GET request timed out retrieving RFC editor errata: {exc}") - return # failed - errata_data = response.json() - if len(index_data) < rfceditor.MIN_INDEX_RESULTS: - log.log("Not enough index entries, only %s" % len(index_data)) - return # failed - if len(errata_data) < rfceditor.MIN_ERRATA_RESULTS: - log.log("Not enough errata entries, only %s" % len(errata_data)) - return # failed - newly_published = set() - for rfc_number, changes, doc, rfc_published in rfceditor.update_docs_from_rfc_index( - index_data, errata_data, skip_older_than_date=skip_date - ): - for c in changes: - log.log("RFC%s, %s: %s" % (rfc_number, doc.name, c)) - if rfc_published: - newly_published.add(rfc_number) - if len(newly_published) > 0: - rsync_rfcs_from_rfceditor_task.delay(list(newly_published)) - - -@shared_task -def rfc_editor_queue_updates_task(): - log.log(f"Updating RFC Editor queue states from {settings.RFC_EDITOR_QUEUE_URL}") - try: - response = requests.get( - settings.RFC_EDITOR_QUEUE_URL, - timeout=30, # seconds - ) - except requests.Timeout as exc: - log.log(f"GET request timed out retrieving RFC editor queue: {exc}") - return # failed - drafts, warnings = parse_queue(io.StringIO(response.text)) - for w in warnings: - log.log(f"Warning: {w}") - - if len(drafts) < MIN_QUEUE_RESULTS: - log.log("Not enough results, only %s" % len(drafts)) - return # failed - - changed, warnings = update_drafts_from_queue(drafts) - for w in warnings: - log.log(f"Warning: {w}") - - for c in changed: - log.log(f"Updated {c}") @shared_task diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 99ca7b0008a..e13dcd19fd7 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -1,17 +1,11 @@ # Copyright The IETF Trust 2012-2026, All Rights Reserved - -import os -import io import json import datetime from unittest import mock import quopri import requests -from dataclasses import dataclass - -from django.conf import settings from django.core.files.base import ContentFile from django.core.files.storage import storages from django.urls import reverse as urlreverse @@ -24,24 +18,21 @@ from ietf.doc.factories import ( WgDraftFactory, RfcFactory, - DocumentAuthorFactory, DocEventFactory, - BcpFactory, WgRfcFactory, + WgRfcFactory, ) from ietf.doc.models import ( Document, DocEvent, DeletedEvent, DocTagName, - RelatedDocument, State, StateDocEvent, ) from ietf.doc.utils import add_state_change_event -from ietf.group.factories import GroupFactory from ietf.person.factories import PersonFactory from ietf.person.models import Person -from ietf.sync import iana, rfceditor, tasks +from ietf.sync import iana, tasks from ietf.sync.errata import ( update_errata_from_rfceditor, get_errata_last_updated, @@ -56,7 +47,6 @@ from ietf.utils.models import DirtyBits from ietf.utils.test_utils import login_testing_unauthorized from ietf.utils.test_utils import TestCase -from ietf.utils.timezone import date_today, RPC_TZINFO class IANASyncTests(TestCase): @@ -306,494 +296,6 @@ def test_notify_page(self): self.assertContains(r, "new changes at") # we don't actually try posting as that would trigger a real run - - -class RFCSyncTests(TestCase): - def write_draft_file(self, name, size): - with io.open(os.path.join(settings.INTERNET_DRAFT_PATH, name), 'w') as f: - f.write("a" * size) - - def test_rfc_index(self): - area = GroupFactory(type_id='area') - draft_doc = WgDraftFactory( - group__parent=area, - states=[('draft-iesg','rfcqueue')], - ad=Person.objects.get(user__username='ad'), - external_url="http://my-external-url.example.com", - note="this is a note", - pages=54, # make sure this is not 42 - ) - DocumentAuthorFactory.create_batch(2, document=draft_doc) - draft_doc.action_holders.add(draft_doc.ad) # not normally set, but add to be sure it's cleared - - RfcFactory(rfc_number=123) - - today = date_today() - - t = ''' - - - BCP0001 - - RFC1234 - RFC2345 - - - - FYI0001 - - RFC1234 - - - - STD0002 - Test - - RFC1234 - - - - RFC1234 - A Testing RFC - - A. Irector - - - %(month)s - %(year)s - - - ASCII - - 42 - - test - -

    This is some interesting text.

     - %(name)s-%(rev)s - - RFC123 - - - BCP0001 - - PROPOSED STANDARD - PROPOSED STANDARD - IETF - %(area)s - %(group)s - http://www.rfc-editor.org/errata_search.php?rfc=1234 -     -    ''' % dict(year=today.strftime("%Y"), - month=today.strftime("%B"), - name=draft_doc.name, - rev=draft_doc.rev, - area=draft_doc.group.parent.acronym, - group=draft_doc.group.acronym) - - errata = [{ - "errata_id":1, - "doc-id":"RFC123", # n.b. this is not the same RFC as in the above index XML! - "errata_status_code":"Verified", - "errata_type_code":"Editorial", - "section": "4.1", - "orig_text":" S: 220-smtp.example.com ESMTP Server", - "correct_text":" S: 220 smtp.example.com ESMTP Server", - "notes":"There are 3 instances of this (one on p. 7 and two on p. 8). \n", - "submit_date":"2007-07-19", - "submitter_name":"Rob Siemborski", - "verifier_id":99, - "verifier_name":None, - "update_date":"2019-09-10 09:09:03"}, - ] - - data = rfceditor.parse_index(io.StringIO(t)) - self.assertEqual(len(data), 1) - rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract = data[0] - - # currently, we only check what we actually use - self.assertEqual(rfc_number, 1234) - self.assertEqual(title, "A Testing RFC") - self.assertEqual(rfc_published_date.year, today.year) - self.assertEqual(rfc_published_date.month, today.month) - self.assertEqual(current_status, "Proposed Standard") - self.assertEqual(updates, ["RFC123"]) - self.assertEqual(set(also), set(["BCP1", "FYI1", "STD2"])) - self.assertEqual(draft, draft_doc.name) - self.assertEqual(wg, draft_doc.group.acronym) - self.assertEqual(has_errata, True) - self.assertEqual(stream, "IETF") - self.assertEqual(pages, "42") - self.assertEqual(abstract, "This is some interesting text.") - - draft_filename = "%s-%s.txt" % (draft_doc.name, draft_doc.rev) - self.write_draft_file(draft_filename, 5000) - - event_count_before = draft_doc.docevent_set.count() - draft_title_before = draft_doc.title - draft_abstract_before = draft_doc.abstract - draft_pages_before = draft_doc.pages - - changes = [] - with mock.patch("ietf.sync.rfceditor.log") as mock_log: - for rfc_number, _, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)): - changes.append({"doc_pk": d.pk, "rfc_published": rfc_published}) # we ignore the actual change list - self.assertEqual(rfc_number, 1234) - if rfc_published: - self.assertEqual(d.type_id, "rfc") - self.assertEqual(d.rfc_number, rfc_number) - else: - self.assertEqual(d.type_id, "draft") - self.assertIsNone(d.rfc_number) - - self.assertFalse(mock_log.called, "No log messages expected") - - draft_doc = Document.objects.get(name=draft_doc.name) - draft_events = draft_doc.docevent_set.all() - self.assertEqual(len(draft_events) - event_count_before, 2) - self.assertEqual(draft_events[0].type, "sync_from_rfc_editor") - self.assertEqual(draft_events[1].type, "changed_action_holders") - self.assertEqual(draft_doc.get_state_slug(), "rfc") - self.assertEqual(draft_doc.get_state_slug("draft-iesg"), "pub") - self.assertCountEqual(draft_doc.action_holders.all(), []) - self.assertEqual(draft_doc.title, draft_title_before) - self.assertEqual(draft_doc.abstract, draft_abstract_before) - self.assertEqual(draft_doc.pages, draft_pages_before) - self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, draft_filename))) - self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, draft_filename))) - - rfc_doc = Document.objects.filter(rfc_number=1234, type_id="rfc").first() - self.assertIsNotNone(rfc_doc, "RFC document should have been created") - self.assertEqual(rfc_doc.author_persons_or_names(), draft_doc.author_persons_or_names()) - rfc_events = rfc_doc.docevent_set.all() - self.assertEqual(len(rfc_events), 8) - expected_events = [ - ["sync_from_rfc_editor", ""], # Not looking for exact desc match here - see detailed tests below - ["sync_from_rfc_editor", "Imported membership of rfc1234 in std2 via sync to the rfc-index"], - ["std_history_marker", "No history of STD2 is currently available in the datatracker before this point"], - ["sync_from_rfc_editor", "Imported membership of rfc1234 in fyi1 via sync to the rfc-index"], - ["fyi_history_marker", "No history of FYI1 is currently available in the datatracker before this point"], - ["sync_from_rfc_editor", "Imported membership of rfc1234 in bcp1 via sync to the rfc-index"], - ["bcp_history_marker", "No history of BCP1 is currently available in the datatracker before this point"], - ["published_rfc", "RFC published"] - ] - for index, [event_type, desc] in enumerate(expected_events): - self.assertEqual(rfc_events[index].type, event_type) - if index == 0: - self.assertIn("Received changes through RFC Editor sync (created document RFC 1234,", rfc_events[0].desc) - self.assertIn(f"created became rfc relationship between {rfc_doc.came_from_draft().name} and RFC 1234", rfc_events[0].desc) - self.assertIn("set title to 'A Testing RFC'", rfc_events[0].desc) - self.assertIn("set abstract to 'This is some interesting text.'", rfc_events[0].desc) - self.assertIn("set pages to 42", rfc_events[0].desc) - self.assertIn("set standardization level to Proposed Standard", rfc_events[0].desc) - self.assertIn(f"added RFC published event at {rfc_events[0].time.astimezone(RPC_TZINFO):%Y-%m-%d}", rfc_events[0].desc) - self.assertIn("created updates relation between RFC 1234 and RFC 123", rfc_events[0].desc) - self.assertIn("added Errata tag", rfc_events[0].desc) - else: - self.assertEqual(rfc_events[index].desc, desc) - self.assertEqual(rfc_events[7].time.astimezone(RPC_TZINFO).date(), today) - for subseries_name in ["bcp1", "fyi1", "std2"]: - sub = Document.objects.filter(type_id=subseries_name[:3],name=subseries_name).first() - self.assertIsNotNone(sub, f"{subseries_name} not created") - self.assertTrue(rfc_doc in sub.contains()) - self.assertTrue(sub in rfc_doc.part_of()) - self.assertEqual(rfc_doc.get_state_slug(), "published") - # Should have an "errata" tag because there is an errata-url in the index XML, but no "verified-errata" tag - # because there is no verified item in the errata JSON with doc-id matching the RFC document. - tag_slugs = rfc_doc.tags.values_list("slug", flat=True) - self.assertTrue("errata" in tag_slugs) - self.assertFalse("verified-errata" in tag_slugs) - # TODO: adjust these when we have subseries document types - # self.assertTrue(DocAlias.objects.filter(name="rfc1234", docs=rfc_doc)) - # self.assertTrue(DocAlias.objects.filter(name="bcp1", docs=rfc_doc)) - # self.assertTrue(DocAlias.objects.filter(name="fyi1", docs=rfc_doc)) - # self.assertTrue(DocAlias.objects.filter(name="std1", docs=rfc_doc)) - self.assertTrue(RelatedDocument.objects.filter(source=rfc_doc, target__name="rfc123", relationship="updates").exists()) - self.assertTrue(RelatedDocument.objects.filter(source=draft_doc, target=rfc_doc, relationship="became_rfc").exists()) - self.assertEqual(rfc_doc.title, "A Testing RFC") - self.assertEqual(rfc_doc.abstract, "This is some interesting text.") - self.assertEqual(rfc_doc.std_level_id, "ps") - self.assertEqual(rfc_doc.pages, 42) - self.assertEqual(rfc_doc.stream, draft_doc.stream) - self.assertEqual(rfc_doc.group, draft_doc.group) - self.assertEqual(rfc_doc.words, draft_doc.words) - self.assertEqual(rfc_doc.ad, draft_doc.ad) - self.assertEqual(rfc_doc.external_url, draft_doc.external_url) - self.assertEqual(rfc_doc.note, draft_doc.note) - - # check that we got the expected changes - self.assertEqual(len(changes), 2) - self.assertEqual(changes[0]["doc_pk"], draft_doc.pk) - self.assertEqual(changes[0]["rfc_published"], False) - self.assertEqual(changes[1]["doc_pk"], rfc_doc.pk) - self.assertEqual(changes[1]["rfc_published"], True) - - # make sure we can apply it again with no changes - changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30))) - self.assertEqual(len(changed), 0) - - def test_rfc_index_subseries_replacement(self): - today = date_today() - author = PersonFactory(name="Some Bozo") - - # Start with two BCPs, each containing an rfc - rfc1, rfc2, rfc3 = RfcFactory.create_batch(3, authors=[author]) - bcp1 = BcpFactory(contains=[rfc1]) - bcp2 = BcpFactory(contains=[rfc2]) - - def _nameify(doc): - """Convert a name like 'rfc1' to 'RFC0001""" - return f"{doc.name[:3].upper()}{int(doc.name[3:]):04d}" - - # RFC index that replaces rfc2 with rfc3 in bcp2 - index_xml = f""" - - - {_nameify(bcp1)} - - {_nameify(rfc1)} - - - - {_nameify(bcp2)} - - {_nameify(rfc3)} - - - - {_nameify(rfc1)} - {rfc1.title} - - Some Bozo - - - {today.strftime('%B')} - {today.strftime('%Y')} - - - ASCII - - 42 - - test - -

    This is some interesting text.

     - - {_nameify(bcp1)} - - PROPOSED STANDARD - PROPOSED STANDARD - IETF -     - - {_nameify(rfc2)} - {rfc2.title} - - Some Bozo - - - {today.strftime('%B')} - {today.strftime('%Y')} - - - ASCII - - 42 - - test - -

    This is some interesting text.

     - PROPOSED STANDARD - PROPOSED STANDARD - IETF -     - - {_nameify(rfc3)} - {rfc3.title} - - Some Bozo - - - {today.strftime('%B')} - {today.strftime('%Y')} - - - ASCII - - 42 - - test - -

    This is some interesting text.

     - - {_nameify(bcp2)} - - PROPOSED STANDARD - PROPOSED STANDARD - IETF -     -    """ - data = rfceditor.parse_index(io.StringIO(index_xml)) # parse index - self.assertEqual(len(data), 3) # check that we parsed 3 RFCs - # Process the data by consuming the generator - for _ in rfceditor.update_docs_from_rfc_index(data, []): - pass - # Confirm that the expected changes were made - self.assertCountEqual(rfc1.related_that("contains"), [bcp1]) - self.assertCountEqual(rfc2.related_that("contains"), []) - self.assertCountEqual(rfc3.related_that("contains"), [bcp2]) - - def _generate_rfc_queue_xml(self, draft, state, auth48_url=None): - """Generate an RFC queue xml string for a draft""" - t = ''' -
    - -%(name)s-%(rev)s.txt -2010-09-08 -%(state)s -%(auth48_url)s - -%(ref)s -IN-QUEUE - -A. Author - -%(title)s - -10000000 -%(group)s - -
    -    ''' % dict(name=draft.name, - rev=draft.rev, - title=draft.title, - group=draft.group.name, - ref="draft-ietf-test", - state=state, - auth48_url=(auth48_url or '')) - t = t.replace('\n', '') # strip empty auth48-url tags - return t - - def test_rfc_queue(self): - draft = WgDraftFactory(states=[('draft-iesg','ann')], ad=Person.objects.get(user__username='ad')) - draft.action_holders.add(draft.ad) # add an action holder so we can test that it's removed later - - expected_auth48_url = "http://www.rfc-editor.org/auth48/rfc1234" - t = self._generate_rfc_queue_xml(draft, - state='EDIT*R*A(1G)', - auth48_url=expected_auth48_url) - - drafts, warnings = rfceditor.parse_queue(io.StringIO(t)) - # rfceditor.parse_queue() is tested independently; just sanity check here - self.assertEqual(len(drafts), 1) - self.assertEqual(len(warnings), 0) - - mailbox_before = len(outbox) - - changed, warnings = rfceditor.update_drafts_from_queue(drafts) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - - draft = Document.objects.get(pk=draft.pk) - self.assertEqual(draft.get_state_slug("draft-rfceditor"), "edit") - self.assertEqual(draft.get_state_slug("draft-iesg"), "rfcqueue") - self.assertCountEqual(draft.action_holders.all(), []) - self.assertEqual(set(draft.tags.all()), set(DocTagName.objects.filter(slug__in=("iana", "ref")))) - events = draft.docevent_set.all() - self.assertEqual(events[0].type, "changed_state") # changed draft-iesg state - self.assertEqual(events[1].type, "changed_action_holders") - self.assertEqual(events[2].type, "changed_state") # changed draft-rfceditor state - self.assertEqual(events[3].type, "rfc_editor_received_announcement") - - self.assertEqual(len(outbox), mailbox_before + 1) - self.assertTrue("RFC Editor queue" in outbox[-1]["Subject"]) - - # make sure we can apply it again with no changes - changed, warnings = rfceditor.update_drafts_from_queue(drafts) - self.assertEqual(len(changed), 0) - self.assertEqual(len(warnings), 0) - - def test_rfceditor_parse_queue(self): - """Test that rfceditor.parse_queue() behaves as expected. - - Currently does a limited test - old comment was - "currently, we only check what we actually use". - """ - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - t = self._generate_rfc_queue_xml(draft, - state='EDIT*R*A(1G)', - auth48_url="http://www.rfc-editor.org/auth48/rfc1234") - - drafts, warnings = rfceditor.parse_queue(io.StringIO(t)) - self.assertEqual(len(drafts), 1) - self.assertEqual(len(warnings), 0) - - draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs = drafts[0] - self.assertEqual(draft_name, draft.name) - self.assertEqual(state, "EDIT") - self.assertEqual(set(tags), set(["iana", "ref"])) - self.assertEqual(auth48, "http://www.rfc-editor.org/auth48/rfc1234") - - def test_rfceditor_parse_queue_TI_state(self): - # Test with TI state introduced 11 Sep 2019 - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - t = self._generate_rfc_queue_xml(draft, - state='TI', - auth48_url="http://www.rfc-editor.org/auth48/rfc1234") - __, warnings = rfceditor.parse_queue(io.StringIO(t)) - self.assertEqual(len(warnings), 0) - - def _generate_rfceditor_update(self, draft, state, tags=None, auth48_url=None): - """Helper to generate fake output from rfceditor.parse_queue()""" - return [[ - draft.name, # draft_name - '2020-06-03', # date_received - state, - tags or [], - '1', # missref_generation - 'ietf', # stream - auth48_url or '', - '', # cluster - ['draft-ietf-test'], # refs - ]] - - def test_update_draft_auth48_url(self): - """Test that auth48 URLs are handled correctly.""" - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - - # Step 1 setup: update to a state with no auth48 URL - changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='EDIT') - ) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() - self.assertIsNone(auth48_docurl) - - # Step 2: update to auth48 state with auth48 URL - changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='AUTH48', auth48_url='http://www.rfc-editor.org/rfc1234') - ) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() - self.assertIsNotNone(auth48_docurl) - self.assertEqual(auth48_docurl.url, 'http://www.rfc-editor.org/rfc1234') - - # Step 3: update to auth48-done state without auth48 URL - changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='AUTH48-DONE') - ) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() - self.assertIsNone(auth48_docurl) class DiscrepanciesTests(TestCase): def test_discrepancies(self): @@ -1075,192 +577,7 @@ def test_mark_errata_as_processed(self): class TaskTests(TestCase): - @override_settings( - RFC_EDITOR_INDEX_URL="https://rfc-editor.example.com/index/", - RFC_EDITOR_ERRATA_JSON_URL="https://rfc-editor.example.com/errata/", - ) - @mock.patch("ietf.sync.tasks.rfceditor.update_docs_from_rfc_index") - @mock.patch("ietf.sync.tasks.rfceditor.parse_index") - @mock.patch("ietf.sync.tasks.requests.get") - @mock.patch("ietf.sync.tasks.rsync_rfcs_from_rfceditor_task.delay") - def test_rfc_editor_index_update_task( - self, rsync_task_mock, requests_get_mock, parse_index_mock, update_docs_mock - ) -> None: # the annotation here prevents mypy from complaining about annotation-unchecked - """rfc_editor_index_update_task calls helpers correctly - - This tests that data flow is as expected. Assumes the individual helpers are - separately tested to function correctly. - """ - @dataclass - class MockIndexData: - """Mock index item that claims to be a specified length""" - length: int - - def __len__(self): - return self.length - - @dataclass - class MockResponse: - """Mock object that contains text and json() that claims to be a specified length""" - text: str - json_length: int = 0 - - def json(self): - return MockIndexData(length=self.json_length) - - # Response objects - index_response = MockResponse(text="this is the index") - errata_response = MockResponse( - text="these are the errata", json_length=rfceditor.MIN_ERRATA_RESULTS - ) - rfc = RfcFactory() - # Test with full_index = False - rsync_task_mock.return_value = None - requests_get_mock.side_effect = (index_response, errata_response) # will step through these - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) - update_docs_mock.return_value = ( - (rfc.rfc_number, ("something changed",), rfc, False), - ) - - tasks.rfc_editor_index_update_task(full_index=False) - - # Check parse_index() call - self.assertTrue(parse_index_mock.called) - (parse_index_args, _) = parse_index_mock.call_args - self.assertEqual( - parse_index_args[0].read(), # arg is a StringIO - "this is the index", - "parse_index is called with the index text in a StringIO", - ) - - # Check update_docs_from_rfc_index call - self.assertTrue(update_docs_mock.called) - (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args - self.assertEqual( - update_docs_args, (parse_index_mock.return_value, errata_response.json()) - ) - self.assertIsNotNone(update_docs_kwargs["skip_older_than_date"]) - - self.assertFalse(rsync_task_mock.called) - - # Test again with full_index = True - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = (index_response, errata_response) # will step through these - tasks.rfc_editor_index_update_task(full_index=True) - - # Check parse_index() call - self.assertTrue(parse_index_mock.called) - (parse_index_args, _) = parse_index_mock.call_args - self.assertEqual( - parse_index_args[0].read(), # arg is a StringIO - "this is the index", - "parse_index is called with the index text in a StringIO", - ) - - # Check update_docs_from_rfc_index call - self.assertTrue(update_docs_mock.called) - (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args - self.assertEqual( - update_docs_args, (parse_index_mock.return_value, errata_response.json()) - ) - self.assertIsNone(update_docs_kwargs["skip_older_than_date"]) - - self.assertFalse(rsync_task_mock.called) - - # Test again where the index would cause a new RFC to come into existance - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = ( - index_response, - errata_response, - ) # will step through these - update_docs_mock.return_value = ( - (rfc.rfc_number, ("something changed",), rfc, True), - ) - tasks.rfc_editor_index_update_task(full_index=True) - self.assertTrue(rsync_task_mock.called) - rsync_task_args, rsync_task_kwargs = rsync_task_mock.call_args - self.assertEqual((([rfc.rfc_number],), {}), (rsync_task_args, rsync_task_kwargs)) - - # Test error handling - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = requests.Timeout # timeout on every get() - tasks.rfc_editor_index_update_task(full_index=False) - self.assertFalse(parse_index_mock.called) - self.assertFalse(update_docs_mock.called) - self.assertFalse(rsync_task_mock.called) - - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = [index_response, requests.Timeout] # timeout second get() - tasks.rfc_editor_index_update_task(full_index=False) - self.assertFalse(update_docs_mock.called) - self.assertFalse(rsync_task_mock.called) - - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = [index_response, errata_response] - # feed in an index that is too short - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS - 1) - tasks.rfc_editor_index_update_task(full_index=False) - self.assertTrue(parse_index_mock.called) - self.assertFalse(update_docs_mock.called) - self.assertFalse(rsync_task_mock.called) - - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = [index_response, errata_response] - errata_response.json_length = rfceditor.MIN_ERRATA_RESULTS - 1 # too short - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) - tasks.rfc_editor_index_update_task(full_index=False) - self.assertFalse(update_docs_mock.called) - self.assertFalse(rsync_task_mock.called) - - @override_settings(RFC_EDITOR_QUEUE_URL="https://rfc-editor.example.com/queue/") - @mock.patch("ietf.sync.tasks.update_drafts_from_queue") - @mock.patch("ietf.sync.tasks.parse_queue") - def test_rfc_editor_queue_updates_task(self, mock_parse, mock_update): - # test a request timeout - self.requests_mock.get("https://rfc-editor.example.com/queue/", exc=requests.exceptions.Timeout) - tasks.rfc_editor_queue_updates_task() - self.assertFalse(mock_parse.called) - self.assertFalse(mock_update.called) - - # now return a value rather than an exception - self.requests_mock.get("https://rfc-editor.example.com/queue/", text="the response") - - # mock returning < MIN_QUEUE_RESULTS values - treated as an error, so no update takes place - mock_parse.return_value = ([n for n in range(rfceditor.MIN_QUEUE_RESULTS - 1)], ["a warning"]) - tasks.rfc_editor_queue_updates_task() - self.assertEqual(mock_parse.call_count, 1) - self.assertEqual(mock_parse.call_args[0][0].read(), "the response") - self.assertFalse(mock_update.called) - mock_parse.reset_mock() - - # mock returning +. MIN_QUEUE_RESULTS - should succeed - mock_parse.return_value = ([n for n in range(rfceditor.MIN_QUEUE_RESULTS)], ["a warning"]) - mock_update.return_value = ([1,2,3], ["another warning"]) - tasks.rfc_editor_queue_updates_task() - self.assertEqual(mock_parse.call_count, 1) - self.assertEqual(mock_parse.call_args[0][0].read(), "the response") - self.assertEqual(mock_update.call_count, 1) - self.assertEqual(mock_update.call_args, mock.call([n for n in range(rfceditor.MIN_QUEUE_RESULTS)])) - @override_settings(IANA_SYNC_CHANGES_URL="https://iana.example.com/sync/") @mock.patch("ietf.sync.tasks.iana.update_history_with_changes") @mock.patch("ietf.sync.tasks.iana.parse_changes_json") diff --git a/ietf/sync/views.py b/ietf/sync/views.py index bd539d4e186..58209c2c6ee 100644 --- a/ietf/sync/views.py +++ b/ietf/sync/views.py @@ -35,7 +35,6 @@ def notify(request, org, notification): known_orgs = { "iana": "IANA", - "rfceditor": "RFC Editor", } if org not in known_orgs: @@ -67,29 +66,13 @@ def notify(request, org, notification): known_notifications = { "protocols": "an added reference to an RFC at the IANA protocols page" % settings.IANA_SYNC_PROTOCOLS_URL, "changes": "new changes at the changes JSON dump" % settings.IANA_SYNC_CHANGES_URL, - "queue": "new changes to queue2.xml" % settings.RFC_EDITOR_QUEUE_URL, - "index": "new changes to rfc-index.xml" % settings.RFC_EDITOR_INDEX_URL, } if notification not in known_notifications: raise Http404 if request.method == "POST": - if notification == "index": - log("Queuing RFC Editor index sync from notify view POST") - # Wrap in on_commit in case a transaction is open - # (As of 2024-11-08, this only runs in a transaction during tests) - transaction.on_commit( - lambda: tasks.rfc_editor_index_update_task.delay() - ) - elif notification == "queue": - log("Queuing RFC Editor queue sync from notify view POST") - # Wrap in on_commit in case a transaction is open - # (As of 2024-11-08, this only runs in a transaction during tests) - transaction.on_commit( - lambda: tasks.rfc_editor_queue_updates_task.delay() - ) - elif notification == "changes": + if notification == "changes": log("Queuing IANA changes sync from notify view POST") # Wrap in on_commit in case a transaction is open # (As of 2024-11-08, this only runs in a transaction during tests) diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index eab1d779fba..9b967c013bf 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -601,7 +601,7 @@ diff --git a/ietf/templates/doc/search/status_columns.html b/ietf/templates/doc/search/status_columns.html index 5ba41bb9c4a..50d70fcb379 100644 --- a/ietf/templates/doc/search/status_columns.html +++ b/ietf/templates/doc/search/status_columns.html @@ -11,7 +11,7 @@ {{ doc.friendly_state|safe }} {% endif %} {% if doc|state:"draft-rfceditor" %} - : {{ doc|state:"draft-rfceditor" }} + : {{ doc|state:"draft-rfceditor" }} {% endif %} {{ doc|auth48_alert_badge }} {{ doc|state_age_colored }} diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index 2d34f8361c1..c878ba49f10 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -83,34 +83,6 @@ def create_default_tasks(self): ), ) - PeriodicTask.objects.get_or_create( - name="Partial sync with RFC Editor index", - task="ietf.sync.tasks.rfc_editor_index_update_task", - kwargs=json.dumps(dict(full_index=False)), - defaults=dict( - enabled=False, - crontab=self.crontabs["every_15m_except_midnight"], # don't collide with full sync - description=( - "Reparse the last _year_ of RFC index entries until " - "https://github.com/ietf-tools/datatracker/issues/3734 is addressed. " - "This takes about 20s on production as of 2022-08-11." - ) - ), - ) - - PeriodicTask.objects.get_or_create( - name="Full sync with RFC Editor index", - task="ietf.sync.tasks.rfc_editor_index_update_task", - kwargs=json.dumps(dict(full_index=True)), - defaults=dict( - enabled=False, - crontab=self.crontabs["daily"], - description=( - "Run an extended version of the rfc editor update to catch changes with backdated timestamps" - ), - ), - ) - PeriodicTask.objects.get_or_create( name="Fetch meeting attendance", task="ietf.stats.tasks.fetch_meeting_attendance_task", diff --git a/ietf/utils/timezone.py b/ietf/utils/timezone.py index e08dfa02f2c..5e82c225dc6 100644 --- a/ietf/utils/timezone.py +++ b/ietf/utils/timezone.py @@ -15,7 +15,7 @@ DEADLINE_TZINFO = ZoneInfo('PST8PDT') # Time zone for dates from the RPC. This value is baked into the timestamps on DocEvents -# of type="published_rfc" - see Document.pub_date() and ietf.sync.refceditor.update_docs_from_rfc_index() +# of type="published_rfc" - see Document.pub_date() and the rfc publication workflow # for more information about how that works. RPC_TZINFO = ZoneInfo('PST8PDT') From ea7406f5ca64dbc6ce0fb85a3c9ca029c4a68ca6 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 1 Jul 2026 07:58:52 +1200 Subject: [PATCH 170/188] fix: consistent counts at proceedings and proceedings/attendees (#10935) * fix: consistent counts at proceedings and proceedings/attendees * fix: reconstitute reg objects directly from person objects --------- Co-authored-by: Jennifer Richards --- ietf/meeting/models.py | 107 +++++++++--- ietf/meeting/tests_models.py | 329 ++++++++++++++++++++++++----------- ietf/meeting/tests_views.py | 30 +++- ietf/meeting/utils.py | 11 -- ietf/meeting/views.py | 27 ++- ietf/nomcom/utils.py | 5 +- 6 files changed, 350 insertions(+), 159 deletions(-) diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 7d9e318aabc..8cf386abf29 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -236,16 +236,24 @@ def get_proceedings_materials(self): document__states__slug='active', document__states__type_id='procmaterials' ).order_by('type__order') - def get_attendance(self): - """Get the meeting attendance from the Registrations + def get_attendees(self, sessions=None): + """Get the meeting attendees from the Registrations - Returns a NamedTuple with onsite and remote attributes. Returns None if the record is unavailable - for this meeting. + Returns a pair of sets containing Person objects for persons attending + the meeting either onsite or online (remote). Returns None if the record + is unavailable for this meeting. + + sessions: optional queryset of Sessions to restrict which Attended records + count toward attendance. Defaults to all sessions at this meeting. + Note: restricting sessions does not affect the registration-based path + (checkedin / attended flags), only the Attended-record path. """ number = self.get_number() if number is None or number < 110: return None - Attendance = namedtuple('Attendance', 'onsite remote') + + if sessions is None: + sessions = self.session_set.all() # MeetingRegistration.attended started conflating badge-pickup and session attendance before IETF 114. # We've separated session attendance off to ietf.meeting.Attended, but need to report attendance at older @@ -254,11 +262,8 @@ def get_attendance(self): # Looking up by registration and attendance records separately and joining in # python is far faster than combining the Q objects in the query (~100x). # Further optimization may be possible, but the queries are tricky... - attended_per_meeting_registration = ( - Q(registration__meeting=self) & ( - Q(registration__attended=True) | - Q(registration__checkedin=True) - ) + attended_per_meeting_registration = Q(registration__meeting=self) & ( + Q(registration__attended=True) | Q(registration__checkedin=True) ) attendees_by_reg = set( Person.objects.filter(attended_per_meeting_registration).values_list( @@ -266,29 +271,81 @@ def get_attendance(self): ) ) - attended_per_meeting_attended = ( - Q(attended__session__meeting=self) - # Note that we are not filtering to plenary, wg, or rg sessions - # as we do for nomcom eligibility - if picking up a badge (see above) - # is good enough, just attending e.g. a training session is also good enough - ) attendees_by_att = set( - Person.objects.filter(attended_per_meeting_attended).values_list( + Person.objects.filter(attended__session__in=sessions).values_list( "pk", flat=True ) ) - - attendees = Person.objects.filter( - pk__in=attendees_by_att | attendees_by_reg + + attendees = Person.objects.filter(pk__in=attendees_by_att | attendees_by_reg) + onsite = set( + attendees.filter( + registration__meeting=self, + registration__tickets__attendance_type__slug="onsite", + ) + ) + remote = set( + attendees.filter( + registration__meeting=self, + registration__tickets__attendance_type__slug="remote", + ) ) - onsite = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='onsite')) - remote = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='remote')) remote.difference_update(onsite) - return Attendance( - onsite=len(onsite), - remote=len(remote) + return (onsite, remote) + + def get_attendance(self): + """Get the meeting attendance from the Registrations + + Returns a NamedTuple with onsite and remote attributes. Returns None if the record is unavailable + for this meeting. + """ + attendees = self.get_attendees() + if attendees is None: + return None + Attendance = namedtuple("Attendance", "onsite remote") + onsite, remote = attendees + return Attendance(onsite=len(onsite), remote=len(remote)) + + def nomcom_eligible_participants(self): + """Return (onsite_pks, remote_pks) as frozensets for nomcom eligibility. + + Onsite: has onsite ticket AND (checked in OR attended a qualifying session). + Remote: has remote ticket AND attended a qualifying plenary/wg/rg session. + The registration attended flag alone is not sufficient for remote. + """ + sessions = self.session_set.filter( + Q(type="plenary") | Q(group__type__in=["wg", "rg"]) + ) + attendees = self.get_attendees(sessions=sessions) + if attendees is None: + onsite_pks = frozenset( + self.registration_set.onsite() + .filter(checkedin=True) + .values_list("person", flat=True) + .distinct() + ) + remote_pks = ( + frozenset( + Attended.objects.filter(session__in=sessions) + .values_list("person", flat=True) + .distinct() + ) + - onsite_pks + ) + return (onsite_pks, remote_pks) + onsite, remote = attendees + onsite_pks = frozenset(p.pk for p in onsite) + # Remote requires actual qualifying session attendance; the registration + # attended flag alone (used as a proxy before Attended records existed) + # is not sufficient. + session_attended_pks = frozenset( + Attended.objects.filter(session__in=sessions) + .values_list("person", flat=True) + .distinct() ) + remote_pks = frozenset(p.pk for p in remote) & session_attended_pks + return (onsite_pks, remote_pks) @property def proceedings_format_version(self): diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py index 869d9ec814e..0b9ce3f607f 100644 --- a/ietf/meeting/tests_models.py +++ b/ietf/meeting/tests_models.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2021-2024, All Rights Reserved # -*- coding: utf-8 -*- """Tests of models in the Meeting application""" + import datetime from unittest.mock import patch @@ -10,7 +11,12 @@ import ietf.meeting.models from ietf.group.factories import GroupFactory, GroupHistoryFactory -from ietf.meeting.factories import MeetingFactory, SessionFactory, AttendedFactory, SessionPresentationFactory +from ietf.meeting.factories import ( + MeetingFactory, + SessionFactory, + AttendedFactory, + SessionPresentationFactory, +) from ietf.meeting.factories import RegistrationFactory from ietf.meeting.models import Session from ietf.utils.test_utils import TestCase @@ -18,105 +24,200 @@ class MeetingTests(TestCase): - def test_get_attendance_pre110(self): - """Pre-110 meetings do not calculate attendance""" - meeting = MeetingFactory(type_id='ietf', number='109') - RegistrationFactory.create_batch(3, meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}) - RegistrationFactory.create_batch(4, meeting=meeting, with_ticket={'attendance_type_id': 'remote'}) - RegistrationFactory.create_batch(5, meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}) - self.assertIsNone(meeting.get_attendance()) - - def test_get_attendance_110(self): - """Look at attendance as captured at 110""" - meeting = MeetingFactory(type_id='ietf', number='110') - - # start with attendees that should be ignored - RegistrationFactory.create_batch(3, meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}, attended=True) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}, attended=False) - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 0) - self.assertEqual(attendance.onsite, 0) - - # add online attendees with at least one who registered but did not attend - RegistrationFactory.create_batch(4, meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=True) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=False) - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 4) - self.assertEqual(attendance.onsite, 0) - - # and the same for onsite attendees - RegistrationFactory.create_batch(5, meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=True) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False) - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 4) - self.assertEqual(attendance.onsite, 5) - - # and once more after removing all the online attendees + def test_get_attendees_pre110(self): + """Pre-110 meetings return None""" + meeting = MeetingFactory(type_id="ietf", number="109") + RegistrationFactory.create_batch( + 3, meeting=meeting, with_ticket={"attendance_type_id": "unknown"} + ) + RegistrationFactory.create_batch( + 4, meeting=meeting, with_ticket={"attendance_type_id": "remote"} + ) + RegistrationFactory.create_batch( + 5, meeting=meeting, with_ticket={"attendance_type_id": "onsite"} + ) + self.assertIsNone(meeting.get_attendees()) + + def test_get_attendees_110(self): + """Registration-based attendance (attended flag) used at IETF 110""" + meeting = MeetingFactory(type_id="ietf", number="110") + + # Unknown-ticket attendees are excluded regardless of attended flag + RegistrationFactory.create_batch( + 3, + meeting=meeting, + with_ticket={"attendance_type_id": "unknown"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "unknown"}, + attended=False, + ) + onsite, remote = meeting.get_attendees() + self.assertEqual(len(onsite), 0) + self.assertEqual(len(remote), 0) + + # Remote attendees: only those with attended=True are included + remote_regs = RegistrationFactory.create_batch( + 4, + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=False, + ) + onsite, remote = meeting.get_attendees() + self.assertEqual({p.pk for p in onsite}, set()) + self.assertEqual({p.pk for p in remote}, {r.person.pk for r in remote_regs}) + + # Onsite attendees: only those with attended=True are included + onsite_regs = RegistrationFactory.create_batch( + 5, + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + ) + onsite, remote = meeting.get_attendees() + self.assertEqual({p.pk for p in onsite}, {r.person.pk for r in onsite_regs}) + self.assertEqual({p.pk for p in remote}, {r.person.pk for r in remote_regs}) + + # Deleting remote registrations empties the remote set meeting.registration_set.remote().delete() - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 0) - self.assertEqual(attendance.onsite, 5) - - def test_get_attendance_113(self): - """Simulate IETF 113 attendance gathering data""" - meeting = MeetingFactory(type_id='ietf', number='113') - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=True, checkedin=False) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=True) - p1 = RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=False).person - AttendedFactory(session__meeting=meeting, person=p1) - p2 = RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=False, checkedin=False).person - AttendedFactory(session__meeting=meeting, person=p2) - attendance = meeting.get_attendance() - self.assertEqual(attendance.onsite, 3) - self.assertEqual(attendance.remote, 1) - - def test_get_attendance_keeps_meetings_distinct(self): + onsite, remote = meeting.get_attendees() + self.assertEqual({p.pk for p in onsite}, {r.person.pk for r in onsite_regs}) + self.assertEqual(len(remote), 0) + + def test_get_attendees_113(self): + """Simulate IETF 113: mix of attended flag, checkedin flag, and Attended records""" + meeting = MeetingFactory(type_id="ietf", number="113") + p_attended = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=True, + checkedin=False, + ).person + p_checkedin = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + checkedin=True, + ).person + p_onsite_session = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + checkedin=False, + ).person + AttendedFactory(session__meeting=meeting, person=p_onsite_session) + p_remote_session = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=False, + checkedin=False, + ).person + AttendedFactory(session__meeting=meeting, person=p_remote_session) + onsite, remote = meeting.get_attendees() + self.assertEqual( + {p.pk for p in onsite}, {p_attended.pk, p_checkedin.pk, p_onsite_session.pk} + ) + self.assertEqual({p.pk for p in remote}, {p_remote_session.pk}) + + def test_get_attendees_keeps_meetings_distinct(self): """No cross-talk between attendance for different meetings""" - # numbers are arbitrary here - first_mtg = MeetingFactory(type_id='ietf', number='114') - second_mtg = MeetingFactory(type_id='ietf', number='115') + first_mtg = MeetingFactory(type_id="ietf", number="114") + second_mtg = MeetingFactory(type_id="ietf", number="115") - # Create a person who attended a remote session for first_mtg and onsite for second_mtg without - # checking in for either. - p = RegistrationFactory(meeting=second_mtg, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=False).person + # Person with remote ticket at first_mtg and onsite ticket at second_mtg, attended both via Attended records + p = RegistrationFactory( + meeting=second_mtg, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + checkedin=False, + ).person + RegistrationFactory( + meeting=first_mtg, + person=p, + with_ticket={"attendance_type_id": "remote"}, + attended=False, + checkedin=False, + ) AttendedFactory(session__meeting=first_mtg, person=p) - RegistrationFactory(meeting=first_mtg, person=p, with_ticket={'attendance_type_id': 'remote'}, attended=False, checkedin=False) AttendedFactory(session__meeting=second_mtg, person=p) - att = first_mtg.get_attendance() - self.assertEqual(att.onsite, 0) - self.assertEqual(att.remote, 1) + first_onsite, first_remote = first_mtg.get_attendees() + self.assertEqual({q.pk for q in first_onsite}, set()) + self.assertEqual({q.pk for q in first_remote}, {p.pk}) + + second_onsite, second_remote = second_mtg.get_attendees() + self.assertEqual({q.pk for q in second_onsite}, {p.pk}) + self.assertEqual({q.pk for q in second_remote}, set()) + + def test_get_attendance(self): + """get_attendance delegates to get_attendees and returns counts as a NamedTuple""" + meeting = MeetingFactory(type_id="ietf", number="120") + onsite_regs = RegistrationFactory.create_batch( + 3, + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=True, + ) + remote_regs = RegistrationFactory.create_batch( + 2, + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=True, + ) + att = meeting.get_attendance() + self.assertIsNotNone(att) + self.assertEqual(att.onsite, len(onsite_regs)) + self.assertEqual(att.remote, len(remote_regs)) - att = second_mtg.get_attendance() - self.assertEqual(att.onsite, 1) - self.assertEqual(att.remote, 0) + old_meeting = MeetingFactory(type_id="ietf", number="109") + self.assertIsNone(old_meeting.get_attendance()) def test_vtimezone(self): # normal time zone that should have a zoneinfo file - meeting = MeetingFactory(type_id='ietf', time_zone='America/Los_Angeles', populate_schedule=False) + meeting = MeetingFactory( + type_id="ietf", time_zone="America/Los_Angeles", populate_schedule=False + ) vtz = meeting.vtimezone() self.assertIsNotNone(vtz) self.assertGreater(len(vtz), 0) # time zone that does not have a zoneinfo file should return None - meeting = MeetingFactory(type_id='ietf', time_zone='Fake/Time_Zone', populate_schedule=False) + meeting = MeetingFactory( + type_id="ietf", time_zone="Fake/Time_Zone", populate_schedule=False + ) vtz = meeting.vtimezone() self.assertIsNone(vtz) # ioerror trying to read zoneinfo should return None - meeting = MeetingFactory(type_id='ietf', time_zone='America/Los_Angeles', populate_schedule=False) - with patch('ietf.meeting.models.io.open', side_effect=IOError): + meeting = MeetingFactory( + type_id="ietf", time_zone="America/Los_Angeles", populate_schedule=False + ) + with patch("ietf.meeting.models.io.open", side_effect=IOError): vtz = meeting.vtimezone() self.assertIsNone(vtz) def test_group_at_the_time(self): - m = MeetingFactory(type_id='ietf', date=date_today() - datetime.timedelta(days=10)) + m = MeetingFactory( + type_id="ietf", date=date_today() - datetime.timedelta(days=10) + ) cached_groups = GroupFactory.create_batch(2) m.cached_groups_at_the_time = {g.pk: g for g in cached_groups} # fake the cache - uncached_group_hist = GroupHistoryFactory(time=datetime_today() - datetime.timedelta(days=30)) - self.assertEqual(m.group_at_the_time(uncached_group_hist.group), uncached_group_hist) + uncached_group_hist = GroupHistoryFactory( + time=datetime_today() - datetime.timedelta(days=30) + ) + self.assertEqual( + m.group_at_the_time(uncached_group_hist.group), uncached_group_hist + ) self.assertIn(uncached_group_hist.group.pk, m.cached_groups_at_the_time) @@ -127,27 +228,36 @@ def test_chat_archive_url(self): meeting__number=120, # needs to use proceedings_format_version > 1 ) with override_settings(): - if hasattr(settings, 'CHAT_ARCHIVE_URL_PATTERN'): + if hasattr(settings, "CHAT_ARCHIVE_URL_PATTERN"): del settings.CHAT_ARCHIVE_URL_PATTERN self.assertEqual(session.chat_archive_url(), session.chat_room_url()) - settings.CHAT_ARCHIVE_URL_PATTERN = 'http://chat.example.com' - self.assertEqual(session.chat_archive_url(), 'http://chat.example.com') - chatlog = SessionPresentationFactory(session=session, document__type_id='chatlog').document + settings.CHAT_ARCHIVE_URL_PATTERN = "http://chat.example.com" + self.assertEqual(session.chat_archive_url(), "http://chat.example.com") + chatlog = SessionPresentationFactory( + session=session, document__type_id="chatlog" + ).document self.assertEqual(session.chat_archive_url(), chatlog.get_href()) # datatracker 8.8.0 rolled out on 2022-07-15. Before that, chat logs were jabber logs hosted at www.ietf.org. - session_with_jabber = SessionFactory(group__acronym='fakeacronym', meeting__date=datetime.date(2022,7,14)) - self.assertEqual(session_with_jabber.chat_archive_url(), 'https://www.ietf.org/jabber/logs/fakeacronym?C=M;O=D') - chatlog = SessionPresentationFactory(session=session_with_jabber, document__type_id='chatlog').document + session_with_jabber = SessionFactory( + group__acronym="fakeacronym", meeting__date=datetime.date(2022, 7, 14) + ) + self.assertEqual( + session_with_jabber.chat_archive_url(), + "https://www.ietf.org/jabber/logs/fakeacronym?C=M;O=D", + ) + chatlog = SessionPresentationFactory( + session=session_with_jabber, document__type_id="chatlog" + ).document self.assertEqual(session_with_jabber.chat_archive_url(), chatlog.get_href()) def test_chat_room_name(self): - session = SessionFactory(group__acronym='xyzzy') - self.assertEqual(session.chat_room_name(), 'xyzzy') - session.type_id = 'plenary' - self.assertEqual(session.chat_room_name(), 'plenary') - session.chat_room = 'fnord' - self.assertEqual(session.chat_room_name(), 'fnord') + session = SessionFactory(group__acronym="xyzzy") + self.assertEqual(session.chat_room_name(), "xyzzy") + session.type_id = "plenary" + self.assertEqual(session.chat_room_name(), "plenary") + session.chat_room = "fnord" + self.assertEqual(session.chat_room_name(), "fnord") def test_alpha_str(self): self.assertEqual(Session._alpha_str(0), "a") @@ -157,7 +267,11 @@ def test_alpha_str(self): self.assertEqual(Session._alpha_str(27 * 26 - 1), "zz") self.assertEqual(Session._alpha_str(27 * 26), "aaa") - @patch.object(ietf.meeting.models.Session, "_session_recording_url_label", return_value="LABEL") + @patch.object( + ietf.meeting.models.Session, + "_session_recording_url_label", + return_value="LABEL", + ) def test_session_recording_url(self, mock): for session_type in ["ietf", "interim"]: session = SessionFactory(meeting__type_id=session_type) @@ -165,20 +279,29 @@ def test_session_recording_url(self, mock): if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"): del settings.MEETECHO_SESSION_RECORDING_URL self.assertIsNone(session.session_recording_url()) - + settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com" - self.assertEqual(session.session_recording_url(), "http://player.example.com") - - settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}" - self.assertEqual(session.session_recording_url(), "http://player.example.com?LABEL") + self.assertEqual( + session.session_recording_url(), "http://player.example.com" + ) + + settings.MEETECHO_SESSION_RECORDING_URL = ( + "http://player.example.com?{session_label}" + ) + self.assertEqual( + session.session_recording_url(), "http://player.example.com?LABEL" + ) - session.meetecho_recording_name="actualname" + session.meetecho_recording_name = "actualname" session.save() - self.assertEqual(session.session_recording_url(), "http://player.example.com?actualname") + self.assertEqual( + session.session_recording_url(), + "http://player.example.com?actualname", + ) def test_session_recording_url_label_ietf(self): session = SessionFactory( - meeting__type_id='ietf', + meeting__type_id="ietf", meeting__date=date_today(), meeting__number="123", group__acronym="acro", @@ -186,15 +309,17 @@ def test_session_recording_url_label_ietf(self): session_time = session.official_timeslotassignment().timeslot.time self.assertEqual( f"IETF123-ACRO-{session_time:%Y%m%d-%H%M}", # n.b., time in label is UTC - session._session_recording_url_label()) + session._session_recording_url_label(), + ) def test_session_recording_url_label_interim(self): session = SessionFactory( - meeting__type_id='interim', + meeting__type_id="interim", meeting__date=date_today(), group__acronym="acro", ) session_time = session.official_timeslotassignment().timeslot.time self.assertEqual( f"IETF-ACRO-{session_time:%Y%m%d-%H%M}", # n.b., time in label is UTC - session._session_recording_url_label()) + session._session_recording_url_label(), + ) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 17988e50be4..eea08be8c77 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -55,7 +55,7 @@ generate_proceedings_content, diff_meeting_schedules, ) -from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting +from ietf.meeting.utils import add_event_info_to_session_qs from ietf.meeting.utils import create_recording, delete_recording, get_next_sequence, bluesheet_data from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose, generate_agenda_data @@ -9440,20 +9440,32 @@ def test_get_next_sequence(self): sequence = get_next_sequence(group,meeting,'recording') self.assertEqual(sequence,1) - def test_participants_for_meeting(self): + def test_nomcom_eligible_participants_for_meeting(self): m = MeetingFactory.create(type_id='ietf') areg = RegistrationFactory(meeting=m, checkedin=True, with_ticket={'attendance_type_id': 'onsite'}) breg = RegistrationFactory(meeting=m, checkedin=False, with_ticket={'attendance_type_id': 'onsite'}) creg = RegistrationFactory(meeting=m, with_ticket={'attendance_type_id': 'remote'}) dreg = RegistrationFactory(meeting=m, with_ticket={'attendance_type_id': 'remote'}) AttendedFactory(session__meeting=m, session__type_id='plenary', person=creg.person) - checked_in, attended = participants_for_meeting(m) - self.assertIn(areg.person.pk, checked_in) - self.assertNotIn(breg.person.pk, checked_in) - self.assertNotIn(areg.person.pk, attended) - self.assertNotIn(breg.person.pk, attended) - self.assertIn(creg.person.pk, attended) - self.assertNotIn(dreg.person.pk, attended) + onsite_pks, remote_pks = m.nomcom_eligible_participants() + self.assertIn(areg.person.pk, onsite_pks) + self.assertNotIn(breg.person.pk, onsite_pks) + self.assertNotIn(areg.person.pk, remote_pks) + self.assertNotIn(breg.person.pk, remote_pks) + self.assertIn(creg.person.pk, remote_pks) + self.assertNotIn(dreg.person.pk, remote_pks) + + def test_nomcom_eligible_participants_remote_requires_session(self): + """Remote registration.attended=True without a qualifying session does not count (>= 110 path)""" + m = MeetingFactory.create(type_id='ietf', number='119') + # Remote with attended=True but no Attended record: should not be eligible + no_session_reg = RegistrationFactory(meeting=m, attended=True, with_ticket={'attendance_type_id': 'remote'}) + # Remote with attended=True AND a qualifying Attended record: should be eligible + session_reg = RegistrationFactory(meeting=m, attended=True, with_ticket={'attendance_type_id': 'remote'}) + AttendedFactory(session__meeting=m, session__type_id='plenary', person=session_reg.person) + onsite_pks, remote_pks = m.nomcom_eligible_participants() + self.assertNotIn(no_session_reg.person.pk, remote_pks) + self.assertIn(session_reg.person.pk, remote_pks) def test_session_attendance(self): meeting = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number='118') diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 10ae0d3667d..ffd37fc363d 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -1362,17 +1362,6 @@ def post_process(doc): doc.save_with_history([e]) -def participants_for_meeting(meeting): - """ Return a tuple (checked_in, attended) - checked_in = queryset of onsite, checkedin participants values_list('person') - attended = queryset of remote participants who attended a session values_list('person') - """ - checked_in = meeting.registration_set.onsite().filter(checkedin=True).values_list('person', flat=True).distinct() - sessions = meeting.session_set.filter(Q(type='plenary') | Q(group__type__in=['wg', 'rg'])) - attended = Attended.objects.filter(session__in=sessions).values_list('person', flat=True).distinct() - return (checked_in, attended) - - def generate_proceedings_content(meeting, force_refresh=False): """Render proceedings content for a meeting and update cache diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 67a81305b49..913c8bd3021 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -107,7 +107,7 @@ from ietf.meeting.utils import preprocess_meeting_important_dates from ietf.meeting.utils import new_doc_for_session, write_doc_for_session from ietf.meeting.utils import get_activity_stats, post_process, create_recording, delete_recording -from ietf.meeting.utils import participants_for_meeting, generate_bluesheet, bluesheet_data, save_bluesheet +from ietf.meeting.utils import generate_bluesheet, bluesheet_data, save_bluesheet from ietf.message.utils import infer_message from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName, CountryName from ietf.utils import markdown @@ -4816,16 +4816,25 @@ def proceedings_attendees(request, num=None): chart_data = None if int(meeting.number) >= 118: - checked_in, attended = participants_for_meeting(meeting) - regs = list(Registration.objects.onsite().filter(meeting__number=num, checkedin=True)) - onsite_count = len(regs) - regs += [ + onsite, remote = meeting.get_attendees() + onsite_count = len(onsite) + remote_count = len(remote) + onsite_pks = frozenset(p.pk for p in onsite) + remote_pks = frozenset(p.pk for p in remote) + + regs = [ + reg + for reg in Registration.objects.onsite() + .filter(meeting__number=num) + .select_related("person") + if reg.person.pk in onsite_pks + ] + [ reg - for reg in Registration.objects.remote().filter(meeting__number=num).select_related('person') - if reg.person.pk in attended and reg.person.pk not in checked_in + for reg in Registration.objects.remote() + .filter(meeting__number=num) + .select_related("person") + if reg.person.pk in remote_pks ] - remote_count = len(regs) - onsite_count - registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name)) country_codes = [r.country_code for r in registrations if r.country_code] diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index a2ab680df6d..46418714880 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -32,7 +32,6 @@ from ietf.person.models import Email, Person from ietf.mailtrigger.utils import gather_address_lists from ietf.meeting.models import Meeting -from ietf.meeting.utils import participants_for_meeting from ietf.utils.pipe import pipe from ietf.utils.mail import send_mail_text, send_mail, get_payload_text from ietf.utils.log import log @@ -743,8 +742,8 @@ def three_of_five_eligible_9389(previous_five, queryset=None): counts = defaultdict(lambda: 0) for meeting in previous_five: - checked_in, attended = participants_for_meeting(meeting) - for id in set(checked_in) | set(attended): + onsite_pks, remote_pks = meeting.nomcom_eligible_participants() + for id in onsite_pks | remote_pks: counts[id] += 1 return queryset.filter(pk__in=[id for id, count in counts.items() if count >= 3]) From eba657a9d04aee815a14d07a6706ffbbdfa0b207 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 30 Jun 2026 15:01:39 -0500 Subject: [PATCH 171/188] fix: Use primary address for ballot email from for non ietf-stream docs (#11119) * fix: Use primary address for ballot email from for non ietf-stream docs * chore: remove unintended whitespace --- ietf/doc/tests_draft.py | 40 ++++++++++++++++++++++++++++++++++++++++ ietf/doc/views_ballot.py | 5 ++++- ietf/nomcom/tests.py | 4 ++-- ietf/person/factories.py | 4 ++-- ietf/person/tests.py | 6 ++++-- 5 files changed, 52 insertions(+), 7 deletions(-) diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index 688b933aad3..7f075ee95ca 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -2836,3 +2836,43 @@ def _post_json(self, url, json_to_post): ]: self.assertIn(snippet, response["text"]) + # Non-IETF stream: from address uses balloter.formatted_email(), not role_email("ad"). + # Give the balloter a chair role (in a different RG) with a distinct email to prove + # that chair role address is not used; formatted_email() uses the primary email. + irtf_doc = RgDraftFactory() + non_ad_balloter = PersonFactory(name="Some Irsgmember") + other_rg = GroupFactory(type_id="rg") + chair_role_email = EmailFactory( + person=non_ad_balloter, address="chair-role@example.com" + ) + RoleFactory( + name_id="chair", + group=other_rg, + person=non_ad_balloter, + email=chair_role_email, + ) + response = _post_json( + self, + url, + { + "post_data": { + "discuss": "", + "comment": "cccccc", + "position": "yes", + "balloter": non_ad_balloter.pk, + "docname": irtf_doc.name, + "cc_choices": [], + "additional_cc": "", + } + }, + ) + self.assertTrue(response["success"]) + from_line = next( + line for line in response["text"].split("\n") if line.startswith("From: ") + ) + # formatted_email() uses the primary email, not the chair role email + self.assertIn(non_ad_balloter.email().address, from_line) + self.assertNotIn(chair_role_email.address, from_line) + for snippet in ["cccccc", non_ad_balloter.plain_name(), irtf_doc.name]: + self.assertIn(snippet, response["text"]) + diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py index 29aadfdb9b7..ec593934131 100644 --- a/ietf/doc/views_ballot.py +++ b/ietf/doc/views_ballot.py @@ -425,7 +425,10 @@ def build_position_email_from_dict(pos_dict): pos=pos_name, blocking_name=blocking_name, settings=settings)) - frm = balloter.role_email("ad").formatted_email() + if doc.stream_id == "ietf": + frm = balloter.role_email("ad").formatted_email() + else: + frm = balloter.formatted_email() if doc.stream_id == "irtf": addrs = gather_address_lists('irsg_ballot_saved',doc=doc) diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index 28152ef79b1..b406dd31528 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -1190,8 +1190,8 @@ def setUp(self): today = datetime_today() t_minus_3 = today - datetime.timedelta(days=3) t_minus_4 = today - datetime.timedelta(days=4) - e1 = EmailFactory(address="nominee1@example.org", person=PersonFactory(name="Nominee 1"), origin='test') - e2 = EmailFactory(address="nominee2@example.org", person=PersonFactory(name="Nominee 2"), origin='test') + e1 = EmailFactory(address="nominee1@example.org", person__name="Nominee 1", origin='test', primary=True) + e2 = EmailFactory(address="nominee2@example.org", person__name="Nominee 2", origin='test', primary=True) n = make_nomineeposition(self.nomcom,e1.person,gen,None) np = n.nomineeposition_set.get(position=gen) np.time = t_minus_3 diff --git a/ietf/person/factories.py b/ietf/person/factories.py index 98756f26c8a..655f25994b3 100644 --- a/ietf/person/factories.py +++ b/ietf/person/factories.py @@ -95,7 +95,7 @@ def default_emails(obj, create, extracted, **kwargs): # pylint: disable=no-self- extracted = True if create and extracted: make_email = getattr(EmailFactory, 'create' if create else 'build') - make_email(person=obj, address=obj.user.email) + make_email(person=obj, address=obj.user.email, primary=True, **kwargs) @factory.post_generation def default_photo(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument @@ -151,7 +151,7 @@ class Meta: django_get_or_create = ('address',) address = factory.Sequence(fake_email_address) - person = factory.SubFactory(PersonFactory) + person = factory.SubFactory(PersonFactory, default_emails=False) active = True primary = False diff --git a/ietf/person/tests.py b/ietf/person/tests.py index f55d8b8a347..42c2c1547cb 100644 --- a/ietf/person/tests.py +++ b/ietf/person/tests.py @@ -78,7 +78,9 @@ def test_ajax_person_email_json(self): def test_default_email(self): person = PersonFactory() - primary = EmailFactory(person=person, primary=True, active=True) + primary = person.email_set.get() + self.assertEqual(primary.primary, True) + self.assertEqual(primary.active, True) EmailFactory(person=person, primary=False, active=True) EmailFactory(person=person, primary=False, active=False) self.assertTrue(primary.address in person.formatted_email()) @@ -358,7 +360,7 @@ def test_get_extra_primary(self): source = PersonFactory() target = PersonFactory() extra = get_extra_primary(source, target) - self.assertTrue(extra == list(source.email_set.filter(primary=True))) + self.assertEqual(set(extra), set(source.email_set.filter(primary=True))) def test_dedupe_aliases(self): person = PersonFactory() From 9f6f99c5957fb4f5cc9d4ad96a39595bacd7bf6f Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 2 Jul 2026 09:32:24 -0500 Subject: [PATCH 172/188] feat: better arguments for manually updating rfc json (#11131) * feat: better arguments for manually updating rfc json * fix: sort and deduplicate generated integer list --- ietf/sync/tasks.py | 24 ++++++++++++++++++- ietf/sync/tests_tasks.py | 17 ++++++++++++++ ietf/sync/tests_utils.py | 47 +++++++++++++++++++++++++++++++++++- ietf/sync/utils.py | 51 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 137 insertions(+), 2 deletions(-) diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index c61e754ed30..3af5cb8984d 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -31,7 +31,12 @@ create_std_txt_index, rfcindex_is_dirty, mark_rfcindex_as_processed, mark_rfcindex_as_dirty, ) -from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper +from ietf.sync.utils import ( + build_from_file_content, + expand_rfc_number_range_list, + load_rfcs_into_blobdb, + rsync_helper, +) from ietf.utils import log @@ -219,6 +224,23 @@ def update_errata_from_rfceditor_task(): if changed_numbers: update_rfc_json_task.delay(list(changed_numbers)) +@shared_task +def update_rfc_json_by_range_list_task(ranges: str) -> None: + """Regenerate RFC JSON for the RFCs described by a range-list string + + See expand_rfc_number_range_list() for the accepted format (e.g. + "[1,100,1000-1004]"). Invalid input is logged and otherwise ignored. + """ + try: + rfc_numbers = expand_rfc_number_range_list(ranges) + except ValueError as e: + log.log( + f"update_rfc_json_by_range_list_task: ignoring invalid input " + f"'{ranges}': {e}" + ) + return + if rfc_numbers: + update_rfc_json_task(rfc_numbers) @shared_task def update_rfc_json_task(rfc_numbers: list[int]) -> None: diff --git a/ietf/sync/tests_tasks.py b/ietf/sync/tests_tasks.py index edfd080079a..57284b72980 100644 --- a/ietf/sync/tests_tasks.py +++ b/ietf/sync/tests_tasks.py @@ -536,3 +536,20 @@ def test_json_task_not_called_when_no_changes( mock_update.return_value = set() tasks.update_errata_from_rfceditor_task() mock_json_delay.assert_not_called() + + +class UpdateRfcJsonByRangeListTaskTests(TestCase): + @mock.patch("ietf.sync.tasks.update_rfc_json_task") + def test_expands_range_list_and_calls_update(self, mock_update): + tasks.update_rfc_json_by_range_list_task("[1,100,1000-1004]") + mock_update.assert_called_once_with([1, 100, 1000, 1001, 1002, 1003]) + + @mock.patch("ietf.sync.tasks.update_rfc_json_task") + def test_invalid_input_is_ignored(self, mock_update): + tasks.update_rfc_json_by_range_list_task("[5-3]") + mock_update.assert_not_called() + + @mock.patch("ietf.sync.tasks.update_rfc_json_task") + def test_empty_input_does_not_call_update(self, mock_update): + tasks.update_rfc_json_by_range_list_task("[]") + mock_update.assert_not_called() diff --git a/ietf/sync/tests_utils.py b/ietf/sync/tests_utils.py index bb4a859e301..ed1dd4a62aa 100644 --- a/ietf/sync/tests_utils.py +++ b/ietf/sync/tests_utils.py @@ -7,10 +7,55 @@ from ietf import settings from ietf.doc.factories import RfcFactory from ietf.doc.storage_utils import exists_in_storage, retrieve_str -from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper +from ietf.sync.utils import ( + build_from_file_content, + expand_rfc_number_range_list, + load_rfcs_into_blobdb, + rsync_helper, +) from ietf.utils.test_utils import TestCase +class ExpandRfcNumberRangeListTests(TestCase): + def test_expands_bare_numbers_and_ranges(self): + # ranges follow Python range() conventions: right value is excluded + self.assertEqual( + expand_rfc_number_range_list("[1,100,1000-1004]"), + [1, 100, 1000, 1001, 1002, 1003], + ) + + def test_accepts_input_without_brackets_and_whitespace(self): + self.assertEqual( + expand_rfc_number_range_list(" 1 , 3 - 5 , 9 "), + [1, 3, 4, 9], + ) + + def test_overlapping_ranges_are_sorted_and_deduplicated(self): + self.assertEqual( + expand_rfc_number_range_list("[5,1-4,3-6,2]"), + [1, 2, 3, 4, 5], + ) + + def test_empty_input_yields_empty_list(self): + self.assertEqual(expand_rfc_number_range_list("[]"), []) + self.assertEqual(expand_rfc_number_range_list(""), []) + + def test_rejects_invalid_input(self): + for bad in [ + "[0]", # zero is not positive + "[-5]", # negatives are not accepted + "[abc]", # non-numeric + "[5-3]", # reversed range + "[5-5]", # empty range (start not less than end) + "[1-]", # missing end + "[-1]", # missing start + "[1-2-3]", # malformed range + "[1.5]", # not an integer + ]: + with self.assertRaises(ValueError, msg=f"expected ValueError for {bad!r}"): + expand_rfc_number_range_list(bad) + + class RsyncHelperTests(TestCase): def test_rsync_helper(self): with ( diff --git a/ietf/sync/utils.py b/ietf/sync/utils.py index b3bdd8d2068..b75d80e76bc 100644 --- a/ietf/sync/utils.py +++ b/ietf/sync/utils.py @@ -14,6 +14,57 @@ def rsync_helper(subprocess_arg_array: list[str]): subprocess.run(["/usr/bin/rsync"]+subprocess_arg_array) + +def _parse_positive_int(value: str) -> int: + """Parse a string as a positive (>= 1) integer, raising ValueError otherwise""" + token = value.strip() + # str.isdigit() rejects signs, whitespace, and non-digits, so anything that + # passes is a non-negative integer literal; guard against zero separately. + if not token.isdigit(): + raise ValueError(f"'{value}' is not a positive integer") + number = int(token) + if number < 1: + raise ValueError(f"'{value}' is not a positive integer") + return number + + +def expand_rfc_number_range_list(ranges: str) -> list[int]: + """Expand a range-list string into a list of RFC numbers + + The string is a comma-separated list of tokens, optionally surrounded by a + pair of square brackets. Each token is either a bare positive integer or a + pair of positive integers separated by a hyphen. A hyphenated pair is + expanded following the convention of Python's range(): the left value is + included and the right value is excluded. For example, "[1,100,1000-1004]" + expands to [1, 100, 1000, 1001, 1002, 1003]. + + The returned list is sorted and deduplicated, so overlapping ranges do not + produce repeated numbers. + + Raises ValueError if the input contains anything other than positive + integers and well-formed (non-reversed) ranges. + """ + numbers: set[int] = set() + stripped = ranges.strip() + if stripped.startswith("[") and stripped.endswith("]"): + stripped = stripped[1:-1] + for raw_token in stripped.split(","): + token = raw_token.strip() + if not token: + continue + if "-" in token: + start_str, _, end_str = token.partition("-") + start = _parse_positive_int(start_str) + end = _parse_positive_int(end_str) + if start >= end: + raise ValueError( + f"'{token}' is not a valid range (start must be less than end)" + ) + numbers.update(range(start, end)) + else: + numbers.add(_parse_positive_int(token)) + return sorted(numbers) + def build_from_file_content(rfc_numbers: list[int]) -> str: types_to_sync = settings.RFC_FILE_TYPES + ("json",) lines = [] From 629061210d9350be37500f98dab2c387d87618da Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Thu, 2 Jul 2026 16:34:50 +0200 Subject: [PATCH 173/188] Ignore the timestamp elements, just check that the data is correct (#11124) --- ietf/stats/tests.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index d8e87417026..57e3b76b39b 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -1,6 +1,5 @@ # Copyright The IETF Trust 2016-2026, All Rights Reserved -import calendar import csv import datetime import io @@ -26,7 +25,6 @@ from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory from ietf.meeting.tests_models import MeetingFactory, RegistrationFactory from ietf.submit.factories import SubmissionFactory -from ietf.utils.timezone import date_today class StatisticsTests(TestCase): @@ -158,16 +156,19 @@ def test_review_stats(self): self.assertTrue(q('.review-stats td:contains("1")')) # check stacked chart - expected_date = date_today().replace(day=1) - expected_js_timestamp = calendar.timegm(expected_date.timetuple()) * 1000 url = urlreverse(ietf.stats.views.review_stats, kwargs={ "stats_type": "time" }) url += "?team={}".format(review_req.team.acronym) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertEqual(json.loads(r.context['data']), [ - {"label": "in time", "color": "#3d22b3", "data": [[expected_js_timestamp, 0]]}, - {"label": "late", "color": "#b42222", "data": [[expected_js_timestamp, 0]]} - ]) + stacked_data = json.loads(r.context['data']) + # Ignore the timestamp elements, just check that the data is correct + self.assertEqual(len(stacked_data), 2) + self.assertEqual(stacked_data[0]['label'], 'in time') + self.assertEqual(stacked_data[0]['color'], '#3d22b3') + self.assertEqual(stacked_data[0]['data'], [[stacked_data[0]['data'][0][0], 0]]) + self.assertEqual(stacked_data[1]['label'], 'late') + self.assertEqual(stacked_data[1]['color'], '#b42222') + self.assertEqual(stacked_data[1]['data'], [[stacked_data[0]['data'][0][0], 0]]) q = PyQuery(r.content) self.assertTrue(q('#stats-time-graph')) @@ -177,7 +178,11 @@ def test_review_stats(self): url += "&completion=not_completed" r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertEqual(json.loads(r.context['data']), [{"color": "#3d22b3", "data": [[expected_js_timestamp, 0]]}]) + non_stacked_data = json.loads(r.context['data']) + # Ignore the timestamp elements, just check that the data is correct + self.assertEqual(len(non_stacked_data), 1) + self.assertEqual(non_stacked_data[0]['color'], '#3d22b3') + self.assertEqual(non_stacked_data[0]['data'], [[non_stacked_data[0]['data'][0][0], 0]]) q = PyQuery(r.content) self.assertTrue(q('#stats-time-graph')) From b11e7041c057c2b475729c74241f425d2813f777 Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Thu, 2 Jul 2026 16:51:23 +0200 Subject: [PATCH 174/188] feat: Add IESG decisions to the IESG group menu (#11050) * Add IESG decisions to the IESG group menu * Minor fix to template + copyright dates --- ietf/group/utils.py | 1 + ietf/iesg/views.py | 34 +++++++++++++++-------- ietf/templates/base/menu.html | 6 ++++ ietf/templates/iesg/review_decisions.html | 8 +++--- ietf/templates/iesg/working_groups.html | 8 +++--- 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/ietf/group/utils.py b/ietf/group/utils.py index 6777ed1933b..b1675de659e 100644 --- a/ietf/group/utils.py +++ b/ietf/group/utils.py @@ -241,6 +241,7 @@ def construct_group_menu_context(request, group, selected, group_type, others): entries.append(("Meetings", urlreverse("ietf.group.views.meetings", kwargs=kwargs))) if group.acronym in ["iesg"]: entries.append(("Working Groups", urlreverse("ietf.iesg.views.working_groups"))) + entries.append(("Decisions", urlreverse("ietf.iesg.views.review_decisions"))) if group.acronym in ["iab", "iesg"]: entries.append(("Statements", urlreverse("ietf.group.views.statements", kwargs=kwargs))) entries.append(("Appeals", urlreverse("ietf.group.views.appeals", kwargs=kwargs))) diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index f03afb9fc1a..af04ad06fd5 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2020, All Rights Reserved +# Copyright The IETF Trust 2007-2026, All Rights Reserved # -*- coding: utf-8 -*- # # Portion Copyright (C) 2008-2009 Nokia Corporation and/or its subsidiary(-ies). @@ -59,6 +59,7 @@ from ietf.doc.models import Document, State, LastCallDocEvent, ConsensusDocEvent, DocEvent, IESG_BALLOT_ACTIVE_STATES from ietf.doc.utils import update_telechat, augment_events_with_revision from ietf.group.models import GroupMilestone, Role +from ietf.group.utils import construct_group_menu_context, get_group_or_404 from ietf.iesg.agenda import agenda_data, agenda_sections, fill_in_agenda_docs, get_agenda_date from ietf.iesg.models import TelechatDate, TelechatAgendaContent from ietf.iesg.utils import get_wg_dashboard_info, telechat_page_count @@ -89,13 +90,15 @@ def review_decisions(request, year=None): #doc_levels = ["exp", "inf"] timeframe = "%s" % year if year else "the past 6 months" + group_type = None + group = get_group_or_404("iesg", group_type) return render(request, 'iesg/review_decisions.html', - dict(events=events, - years=years, - year=year, - timeframe=timeframe), - ) + construct_group_menu_context(request, group, "decisions", group_type, { + "events": events, + "years": years, + "year": year, + "timeframe": timeframe})) def agenda_json(request, date=None): data = agenda_data(date) @@ -632,8 +635,17 @@ def working_groups(request): area_summary, area_totals, ad_summary, noad_summary, ad_totals, noad_totals, totals, wg_summary = get_wg_dashboard_info() - return render( - request, - "iesg/working_groups.html", - dict(area_summary=area_summary, area_totals=area_totals, ad_summary=ad_summary, noad_summary=noad_summary, ad_totals=ad_totals, noad_totals=noad_totals, totals=totals, wg_summary=wg_summary), - ) + group_type = None + group = get_group_or_404("iesg", group_type) + + return render(request, 'iesg/working_groups.html', + construct_group_menu_context(request, group, "working groups", group_type, { + "area_summary": area_summary, + "area_totals": area_totals, + "ad_summary": ad_summary, + "noad_summary": noad_summary, + "ad_totals": ad_totals, + "noad_totals": noad_totals, + "totals": totals, + "wg_summary": wg_summary, + })) diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html index b6b744ec21e..905c337850e 100644 --- a/ietf/templates/base/menu.html +++ b/ietf/templates/base/menu.html @@ -132,6 +132,12 @@ IESG dashboard
    • +
  • + + IESG decisions + +
  • diff --git a/ietf/templates/iesg/review_decisions.html b/ietf/templates/iesg/review_decisions.html index 68a9d7fb953..2f69c1809b4 100644 --- a/ietf/templates/iesg/review_decisions.html +++ b/ietf/templates/iesg/review_decisions.html @@ -1,8 +1,8 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{% extends "group/group_base.html" %} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin %} -{% block title %}IESG review decisions in {{ timeframe }}{% endblock %} -{% block content %} +{% block group_subtitle %}IESG review decisions in {{ timeframe }}{% endblock %} +{% block group_content %} {% origin %}

    IESG review decisions in {{ timeframe }}

    diff --git a/ietf/templates/iesg/working_groups.html b/ietf/templates/iesg/working_groups.html index b799636857f..1bde1362c50 100644 --- a/ietf/templates/iesg/working_groups.html +++ b/ietf/templates/iesg/working_groups.html @@ -1,11 +1,11 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "group/group_base.html" %} +{# Copyright The IETF Trust 2025-2026, All Rights Reserved #} {% load origin static %} {% block pagehead %} {% endblock %} -{% block title %}IESG view of working groups{% endblock %} -{% block content %} +{% block group_subtitle %}IESG view of working groups{% endblock %} +{% block group_content %} {% origin %}

    IESG view of working groups

    Area Size and Load

    From e7360821d7da140b8348db42d11a8957e0cef195 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 6 Jul 2026 15:58:50 -0300 Subject: [PATCH 175/188] fix: 405 instead of 501 on POST to v1 detail API (#11162) * fix: 405 instead of 501 on detail API POST * fix: import custom ModelResource * fix: use custom MakeResources in mgmt cmd --- ietf/api/__init__.py | 4 +++ ietf/api/management/commands/makeresources.py | 6 ++-- ietf/api/tests.py | 31 +++++++++++++++++++ ietf/review/resources.py | 4 +-- ietf/stats/resources.py | 4 +-- 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index 00733717c4f..f4bfe8330ee 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -10,6 +10,7 @@ from django.apps import apps as django_apps from django.core.exceptions import ObjectDoesNotExist +from django.http import HttpResponseNotAllowed from django.utils.module_loading import autodiscover_modules @@ -50,6 +51,9 @@ def autodiscover(): class ModelResource(tastypie.resources.ModelResource): + def post_detail(self, request, **kwargs): + return HttpResponseNotAllowed(["GET"]) + def generate_cache_key(self, *args, **kwargs): """ Creates a unique-enough cache key. diff --git a/ietf/api/management/commands/makeresources.py b/ietf/api/management/commands/makeresources.py index 889b2cdfb51..8e2c6f4b5cf 100644 --- a/ietf/api/management/commands/makeresources.py +++ b/ietf/api/management/commands/makeresources.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2014-2020, All Rights Reserved +# Copyright The IETF Trust 2014-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,7 +15,7 @@ from django.template import Template, Context from django.utils import timezone -from tastypie.resources import ModelResource +from ietf.api import ModelResource resource_head_template = """# Copyright The IETF Trust {{date}}, All Rights Reserved @@ -23,7 +23,7 @@ # Generated by the makeresources management command {{date}} -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 887969cec15..87f7d684d32 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -12,6 +12,7 @@ from importlib import import_module from pathlib import Path from random import randrange +from urllib.parse import urljoin from django.apps import apps from django.conf import settings @@ -1564,6 +1565,36 @@ def test_serializer_to_etree_handles_xml_invalid_control_chars(self): f"containing control character U+{ord(ch):04X}" ) + def test_post_detail_is_not_allowed(self): + """POST to a detail route returns 405 + + Added because default TastyPie behavior is a 500 due to a NotImplemented + exception. + """ + r = self.client.get("/api/v1", headers={"Accept": "application/json"}) + self.assertValidJSONResponse(r) + resource_list = r.json() + for name in self.apps: + r = self.client.get( + resource_list[name]["list_endpoint"], + headers={"Accept": "application/json"}, + ) + self.assertValidJSONResponse(r) + app_resources = r.json() + model_list = apps.get_app_config(name).get_models() + for model in model_list: + model_name = model._meta.model_name + assert model_name + detail_url = urljoin( + app_resources[model_name]["list_endpoint"], "some-id/" + ) + r = self.client.post(detail_url) + self.assertEqual( + r.status_code, + 405, + f"POST to {name}.{model_name} detail should return 405 status", + ) + class RfcdiffSupportTests(TestCase): diff --git a/ietf/review/resources.py b/ietf/review/resources.py index f79d6dfc221..546ab50a5ca 100644 --- a/ietf/review/resources.py +++ b/ietf/review/resources.py @@ -1,9 +1,9 @@ -# Copyright The IETF Trust 2016-2019, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2016-06-14 04:21 PDT -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache diff --git a/ietf/stats/resources.py b/ietf/stats/resources.py index 59722c505ef..21e2c171acc 100644 --- a/ietf/stats/resources.py +++ b/ietf/stats/resources.py @@ -1,9 +1,9 @@ -# Copyright The IETF Trust 2017-2019, All Rights Reserved +# Copyright The IETF Trust 2017-2026, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2017-02-15 10:10 PST -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache From 1fe544a01a11feb9b13d9df5594054f1cba5b534 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 7 Jul 2026 13:20:50 -0300 Subject: [PATCH 176/188] feat: generate RFC json at publication time (#11143) * feat: generate RFC json at publication time Deprecates uploading json files and ignores the file, instead generating its own. * refactor: partial() instead of lambda in on_commit Using partial() is safer as general practice because it binds arguments at call-time. * test: json upload ignored, creation task queued --- ietf/api/serializers_rpc.py | 2 +- ietf/api/tests_views_rpc.py | 29 +++++++++++++++++++++-------- ietf/api/views_rpc.py | 14 +++++++++++--- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 0e45ee3b391..ca605fe6441 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -805,7 +805,7 @@ class RfcFileSerializer(serializers.Serializer): # works. allowed_extensions = ( ".html", - ".json", + ".json", # deprecated - accepted but ignored, datatracker generates its own ".notprepped.xml", ".pdf", ".txt", diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 72e0d197857..5748a7caa60 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -224,6 +224,7 @@ def test_notify_rfc_published(self, mock_task_delay): self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.update_rfc_json_task") @mock.patch("ietf.api.views_rpc.rebuild_reference_relations_task") @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task") @mock.patch("ietf.api.views_rpc.trigger_red_precomputer_task") @@ -232,6 +233,7 @@ def test_upload_rfc_files( mock_trigger_red_task, mock_update_searchindex_task, mock_rebuild_relations, + mock_update_rfc_json, ): def _valid_post_data(): """Generate a valid post data dict @@ -292,7 +294,7 @@ def _valid_post_data(): ContentFile(b"", "myfile.txt"), ContentFile(b"", "myfile.html"), ContentFile(b"", "myfile.pdf"), - ContentFile(b"", "myfile.json"), + ContentFile(b"", "myfile.json"), # deprecated! ContentFile(b"", "myfile.notprepped.xml"), ] }, @@ -346,18 +348,19 @@ def _valid_post_data(): # valid post mock_trigger_red_task.delay.reset_mock() - r = self.client.post( - url, - _valid_post_data(), - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) + with self.captureOnCommitCallbacks(execute=True): + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) self.assertEqual(r.status_code, 200) self.assertEqual( mock_update_searchindex_task.delay.call_args, mock.call(rfc.rfc_number), ) - for extension in ["xml", "txt", "html", "pdf", "json"]: + for extension in ["xml", "txt", "html", "pdf"]: # no "json"! filename = f"{rfc.name}.{extension}" self.assertEqual( (rfc_path / filename).read_text(), @@ -389,6 +392,12 @@ def _valid_post_data(): b"This is .notprepped.xml", ".notprepped.xml blob should contain the expected content", ) + # special case for json (deprecated and should be ignored) + self.assertFalse((rfc_path / f"{rfc.name}.json").exists()) + self.assertFalse( + Blob.objects.filter(bucket="rfc", name=f"json/{rfc.name}.json").exists() + ) + # Confirm that the red precomputer was triggered correctly self.assertTrue(mock_trigger_red_task.delay.called) _, mock_kwargs = mock_trigger_red_task.delay.call_args @@ -404,6 +413,10 @@ def _valid_post_data(): _, mock_kwargs = mock_rebuild_relations.delay.call_args self.assertIn("doc_names", mock_kwargs) self.assertEqual(mock_kwargs["doc_names"], [rfc.name]) + # Confirm rfc JSON creation was triggered correctly + self.assertTrue(mock_update_rfc_json.delay.called) + mock_args, _ = mock_update_rfc_json.delay.call_args + self.assertEqual(mock_args[0], [rfc.rfc_number]) # re-post with replace = False should now fail mock_update_searchindex_task.reset_mock() diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 0c9e98e2dc0..21d59e3f14c 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2023-2026, All Rights Reserved import os import shutil +from functools import partial from pathlib import Path from tempfile import TemporaryDirectory @@ -463,7 +464,9 @@ def post(self, request): {d.rfc_number for d in rfc.related_that_doc(("updates", "obs"))} ) if related_numbers: - transaction.on_commit(lambda: update_rfc_json_task.delay(related_numbers)) + # Only update json for related rfcs. We can't generate json for _this_ rfc + # until we receive the files and know what formats we have. + transaction.on_commit(partial(update_rfc_json_task.delay, related_numbers)) return Response(NotificationAckSerializer().data) @@ -523,6 +526,10 @@ def post(self, request): for upfile in uploaded_files: uploaded_filename = Path(upfile.name) # name supplied by request uploaded_ext = "".join(uploaded_filename.suffixes) + # Ignore json files, which are deprecated. Remove this when purple no + # longer tries to send them. + if uploaded_ext == ".json": + continue tempfile_path = tmpfile_stem.with_suffix(uploaded_ext) with tempfile_path.open("wb") as dest: for chunk in upfile.chunks(): @@ -565,9 +572,10 @@ def post(self, request): trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating)) # Trigger search index update update_rfc_searchindex_task.delay(rfc.rfc_number) - # Trigger reference relation srebuild + # Trigger reference relations rebuild rebuild_reference_relations_task.delay(doc_names=[rfc.name]) - + # Build rfc json (json for related rfcs was updated in RfcPubNotificationView) + transaction.on_commit(partial(update_rfc_json_task.delay, [rfc.rfc_number])) return Response(NotificationAckSerializer().data) From bd24d4da64d68f81937202d80b04422c81aaf412 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 8 Jul 2026 04:36:55 +1200 Subject: [PATCH 177/188] feat: Expose idnits v3 for everyone (#11166) --- ietf/templates/doc/document_draft.html | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index 9b967c013bf..57194144989 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -666,17 +666,14 @@     Nits - {% if user|has_role:"Area Director" %} - {# IDNITS3 is an experimental service, so only show it to Area Directors #} - Nits-v3 (Experimental) + Nits v3 - {% endif %} Date: Tue, 7 Jul 2026 16:34:14 -0300 Subject: [PATCH 178/188] chore: add keywords to red-content search preset (#11159) --- ietf/utils/searchindex.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py index 4e1ee27895b..3c6e1b8f26f 100644 --- a/ietf/utils/searchindex.py +++ b/ietf/utils/searchindex.py @@ -383,9 +383,9 @@ def update_or_create_rfc_entries( }, "red-content": { "collection": "docs", - "infix": "off,always,off,off", - "query_by": "rfc,filename,authors,content", - "query_by_weights": "127,50,5,1" + "infix": "off,always,off,off,off", + "query_by": "rfc,filename,keywords,authors,content", + "query_by_weights": "127,50,20,5,1" }, } From 6e3549a0b7707ce1f3cfe70e21aafdd4e076a8d4 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 8 Jul 2026 14:22:26 -0300 Subject: [PATCH 179/188] fix: prevent overlap in html page breaks (#11172) --- ietf/static/css/document_html.scss | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ietf/static/css/document_html.scss b/ietf/static/css/document_html.scss index 47ef8d64b47..5a4282195f9 100644 --- a/ietf/static/css/document_html.scss +++ b/ietf/static/css/document_html.scss @@ -205,10 +205,6 @@ display: inline; } - .newpage { - margin-top: -1.25em; - } - } tbody.meta tr { From bd600888d0da085af2eac35d4ad780c33af0d813 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 Jul 2026 17:57:37 -0300 Subject: [PATCH 180/188] test: fix test_submit_bad_author_email() (#11186) Original test accidentally created a second primary email by trying to modify the PK of an existing Email. That led to a stochastic apparently data-dependent failure that depended on which address was returned by `first()`. --- ietf/submit/tests.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index ad361d31b26..abe23c1a643 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -40,7 +40,7 @@ from ietf.meeting.factories import MeetingFactory from ietf.name.models import DraftSubmissionStateName, FormalLanguageName from ietf.person.models import Person -from ietf.person.factories import UserFactory, PersonFactory +from ietf.person.factories import UserFactory, PersonFactory, EmailFactory from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactory from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm from ietf.submit.models import Submission, Preapproval, SubmissionExtResource @@ -1872,10 +1872,8 @@ def test_submit_bad_author_email(self): name = "draft-authorname-testing-bademail" rev = "00" - author = PersonFactory() - email = author.email_set.first() - email.address = '@bad.email' - email.save() + author = PersonFactory(default_emails=False) + EmailFactory(person=author, primary=True, address="@bad.email") status_url, _ = self.do_submission(name=name, rev=rev, author=author, formats=('xml',)) r = self.client.get(status_url) From 8d7139b0538605033a8577174ee5aefd585ad145 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 Jul 2026 18:24:35 -0300 Subject: [PATCH 181/188] fix: use RfcAuthor in bibtex (w/titlepage_name) (#10947) * test: create RfcAuthors in RfcFactory * test: update test to check authors * fix: use RfcAuthor in bibtex (w/titlepage_name) * fix: trailing / on bibtex url * fix: update URL in bibtex warning for legacy RFCs * test: fix failing tests Remove a case that tests fallback from RfcAuthor to DocumentAuthor since we now always have RfcAuthors. Remove RfcAuthor records when deleting Persons they refer to, since we don't allow cascade on this model. --- ietf/doc/factories.py | 7 +++++++ ietf/doc/templatetags/ietf_filters.py | 2 +- ietf/doc/tests.py | 23 +++++++++++++++-------- ietf/nomcom/tests.py | 18 ++++-------------- ietf/templates/doc/document_bibtex.bib | 6 +++--- 5 files changed, 30 insertions(+), 26 deletions(-) diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index 1a178c6f31d..0cf83e4525f 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -128,6 +128,13 @@ def states(obj, create, extracted, **kwargs): else: obj.set_state(State.objects.get(type_id='rfc',slug='published')) + @factory.post_generation + def authors(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument + # Override base class, creating RfcAuthor instead of DocumentAuthor + if create and extracted: + for person in extracted: + RfcAuthorFactory(document=obj, person=person) + class IndividualDraftFactory(BaseDocumentFactory): diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index ae5df641c29..e72cc04ff34 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -138,7 +138,7 @@ def prettystdname(string, space=" "): @register.filter def rfceditor_info_url(rfcnum : str): """Link to the RFC editor info page for an RFC""" - return urljoin(settings.RFC_EDITOR_INFO_BASE_URL, f'rfc{rfcnum}') + return urljoin(settings.RFC_EDITOR_INFO_BASE_URL, f'rfc{rfcnum}/') def doc_name(name): diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index ff4461d4666..86731000730 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -36,9 +36,9 @@ import debug # pyflakes:ignore -from ietf.doc.models import ( Document, DocRelationshipName, RelatedDocument, State, - DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent, BallotType, - EditedAuthorsDocEvent, StateType) +from ietf.doc.models import (Document, DocRelationshipName, RelatedDocument, State, + DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent, BallotType, + EditedAuthorsDocEvent, StateType, RfcAuthor) from ietf.doc.factories import (DocumentFactory, DocEventFactory, CharterFactory, ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, @@ -73,7 +73,7 @@ from ietf.utils.mail import get_payload_text, outbox, empty_outbox from ietf.utils.test_utils import login_testing_unauthorized, unicontent from ietf.utils.test_utils import TestCase -from ietf.utils.text import normalize_text +from ietf.utils.text import normalize_text, texescape from ietf.utils.timezone import date_today, datetime_today, DEADLINE_TZINFO, RPC_TZINFO from ietf.doc.utils_search import AD_WORKLOAD @@ -2133,9 +2133,11 @@ def test_document_bibtex(self): doc = factory() url = urlreverse("ietf.doc.views_doc.document_bibtex", kwargs=dict(name=doc.name)) r = self.client.get(url) - self.assertEqual(r.status_code, 404) + self.assertEqual(r.status_code, 404) + authors = PersonFactory.create_batch(2) rfc = WgRfcFactory.create( - time=datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo(settings.TIME_ZONE)) + time=datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo(settings.TIME_ZONE)), + authors=authors, ) num = rfc.rfc_number DocEventFactory.create( @@ -2152,7 +2154,12 @@ def test_document_bibtex(self): self.assertEqual(entry["doi"], "10.17487/RFC%s" % num) self.assertEqual(entry["year"], "2010") self.assertEqual(entry["month"].lower()[0:3], "oct") - self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}") + self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}/") + escaped_author_names = [ + texescape(ra.titlepage_name) + for ra in RfcAuthor.objects.filter(document=rfc) + ] + self.assertEqual(entry["author"], " and ".join(escaped_author_names)) # self.assertNotIn("day", entry) @@ -2188,7 +2195,7 @@ def test_document_bibtex(self): self.assertEqual(entry["year"], "1990") self.assertEqual(entry["month"].lower()[0:3], "apr") self.assertEqual(entry["day"], "1") - self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}") + self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}/") draft = IndividualDraftFactory.create() docname = "%s-%s" % (draft.name, draft.rev) diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index b406dd31528..ac86d7cf4b4 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -34,6 +34,7 @@ RfcAuthorFactory, WgDraftFactory, WgRfcFactory, ) +from ietf.doc.models import RfcAuthor from ietf.group.factories import GroupFactory, GroupHistoryFactory, RoleFactory, RoleHistoryFactory from ietf.group.models import Group, Role from ietf.meeting.factories import MeetingFactory, AttendedFactory, RegistrationFactory @@ -2535,19 +2536,6 @@ def test_get_qualified_author_queryset(self): people, # does not include extra_person! ) - # Now add an RfcAuthor for only one of the two authors to the RFC. This should - # remove the other author from the eligibility list because the DocumentAuthor - # records are no longer used. - RfcAuthorFactory( - document=rfc, - person=people[0], - titlepage_name="P. Zero", - ) - self.assertCountEqual( - get_qualified_author_queryset(base_qs, now - 5 * one_year, now), - [people[0]], - ) - class rfc8713EligibilityTests(TestCase): @@ -2875,7 +2863,9 @@ def test_elig_by_author(self): self.assertFalse(is_eligible(person,nomcom)) self.assertEqual(set(list_eligible(nomcom=nomcom)),set(eligible)) - Person.objects.filter(pk__in=[p.pk for p in eligible.union(ineligible)]).delete() + people_pks_to_delete = [p.pk for p in eligible.union(ineligible)] + RfcAuthor.objects.filter(person__pk__in=people_pks_to_delete).delete() + Person.objects.filter(pk__in=people_pks_to_delete).delete() class rfc9389EligibilityTests(TestCase): diff --git a/ietf/templates/doc/document_bibtex.bib b/ietf/templates/doc/document_bibtex.bib index 5e52ec3c58d..71b7498351a 100644 --- a/ietf/templates/doc/document_bibtex.bib +++ b/ietf/templates/doc/document_bibtex.bib @@ -6,8 +6,8 @@ {% if doc.type_id == "rfc" %} {% if doc.stream|slugify == "legacy" %} % Datatracker information for RFCs on the Legacy Stream is unfortunately often -% incorrect. Please correct the bibtex below based on the information in the -% actual RFC at https://rfc-editor.org/rfc/rfc{{ doc.rfc_number }}.txt +% incorrect. Please correct the bibtex below based on the information available +% at {{ doc.rfc_number|rfceditor_info_url }} {% endif %} @misc{% templatetag openbrace %}rfc{{ doc.rfc_number }}, series = {Request for Comments}, @@ -25,7 +25,7 @@ @techreport{ publisher = {% templatetag openbrace %}Internet Engineering Task Force{% templatetag closebrace %}, note = {% templatetag openbrace %}Work in Progress{% templatetag closebrace %}, url = {% templatetag openbrace %}{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name=doc.name rev=doc.rev %}{% templatetag closebrace %},{% endif %} - author = {% templatetag openbrace %}{% for author in doc.documentauthor_set.all %}{{ author.person.name|texescape}}{% if not forloop.last %} and {% endif %}{% endfor %}{% templatetag closebrace %}, + author = {% templatetag openbrace %}{% for author in doc.author_persons_or_names %}{% firstof author.titlepage_name|texescape author.person.name|texescape %}{% if not forloop.last %} and {% endif %}{% endfor %}{% templatetag closebrace %}, title = {% templatetag openbrace %}{% templatetag openbrace %}{{doc.title|texescape}}{% templatetag closebrace %}{% templatetag closebrace %}, pagetotal = {{ doc.pages }}, year = {{ doc.pub_date.year }}, From 8ee78c401ca68c6141924700e48f21b6ddde38c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:51:02 -0300 Subject: [PATCH 182/188] chore(deps): bump codecov/codecov-action from 6 to 7 (#11000) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 6 to 7. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v6...v7) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/reusable-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml index 10d04c8265e..852e2d6fedd 100644 --- a/.github/workflows/reusable-tests.yml +++ b/.github/workflows/reusable-tests.yml @@ -75,7 +75,7 @@ jobs: path: geckodriver.log - name: Upload Coverage Results to Codecov - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7 with: disable_search: true files: coverage.xml From ce86eae4f608110c83869b5e12d9d98cd290d7b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:56:32 -0300 Subject: [PATCH 183/188] chore(deps): bump actions/dependency-review-action from 4 to 5 (#10842) Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 4 to 5. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dependency-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 7393e83b829..5bb26e4f6fa 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -17,6 +17,6 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@v7 - name: 'Dependency Review' - uses: actions/dependency-review-action@v4 + uses: actions/dependency-review-action@v5 with: vulnerability-check: false From cf139acf19ec6257033dfbfe1f2b7f91dd9b2015 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:58:08 -0300 Subject: [PATCH 184/188] chore(deps): update pydyf requirement from >=0.11.0 to >=0.12.1 (#10762) Updates the requirements on [pydyf](https://github.com/CourtBouillon/pydyf) to permit the latest version. - [Release notes](https://github.com/CourtBouillon/pydyf/releases) - [Changelog](https://github.com/CourtBouillon/pydyf/blob/main/docs/changelog.rst) - [Commits](https://github.com/CourtBouillon/pydyf/compare/v0.11.0...v0.12.1) --- updated-dependencies: - dependency-name: pydyf dependency-version: 0.12.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 31e8ea69d1e..9f3ddf53b88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -64,7 +64,7 @@ opentelemetry-exporter-otlp-proto-http>=1.38.0 pillow>=11.3.0 psycopg2>=2.9.10 pyang>=2.6.1 -pydyf>=0.11.0 +pydyf>=0.12.1 pyflakes>=3.4.0 pyopenssl>=25.1.0 # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency pyquery>=2.0.1 From 0dcb849b372581022e630b98a344aca9c55473a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:58:53 -0300 Subject: [PATCH 185/188] chore(deps): update lxml requirement from >=6.0.0 to >=6.1.0 (#10760) Updates the requirements on [lxml](https://github.com/lxml/lxml) to permit the latest version. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-6.0.0...lxml-6.1.0) --- updated-dependencies: - dependency-name: lxml dependency-version: 6.1.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9f3ddf53b88..226c960b196 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,7 +48,7 @@ jsonfield>=3.2.0 # deprecated - need to replace with Django's JSONField jsonschema[format]>=4.25.0 jwcrypto>=1.5.6 # for signed notifications - this is aspirational, and is not really used. logging_tree>=1.10 # Used only by the showloggers management command -lxml>=6.0.0 +lxml>=6.1.0 markdown>=3.8.0 types-markdown>=3.8.0 mock>=5.2.0 # should replace with unittest.mock and remove dependency From 133a406da263e5fd73239368dfafcfc1d15adb2a Mon Sep 17 00:00:00 2001 From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:10:21 +0000 Subject: [PATCH 186/188] ci: update base image target version to 20260711T0159 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 72bdb7a7ae1..3738d2ffa8b 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260605T2314 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260711T0159 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 7ca47fd1979..6061a016b79 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260605T2314 +20260711T0159 From b88ff91551dd763ea0a45fb2c2ed25f87979f3ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:40:42 -0300 Subject: [PATCH 187/188] chore(deps): update types-zxcvbn requirement (#10701) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 226c960b196..485e4eb4a3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -92,4 +92,4 @@ weasyprint>=66.0 xml2rfc>=3.30.0 xym>=0.6,<0.10.0 zxcvbn>=4.5.0 -types-zxcvbn~=4.5.0.20250223 # match zxcvbn version +types-zxcvbn~=4.5.0.20260518 # match zxcvbn version From b30026de6b42b0b54b3310e9f8dc2dc448ee7cbb Mon Sep 17 00:00:00 2001 From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:54:04 +0000 Subject: [PATCH 188/188] ci: update base image target version to 20260711T0341 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 3738d2ffa8b..aa1ab4a55cf 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260711T0159 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260711T0341 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 6061a016b79..2fe337efb27 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260711T0159 +20260711T0341