From 3c12880650b6f468ab41ec5782f791c3e019b6ef Mon Sep 17 00:00:00 2001 From: Russ Housley Date: Wed, 5 Nov 2025 15:27:52 -0500 Subject: [PATCH 001/158] feat: Add tab for IESG view of working groups (#9841) * Add tab for WGs. Fixes#9730. * Update utils.py Only add WG tabe for the IESG (not the IAB) --- ietf/group/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ietf/group/utils.py b/ietf/group/utils.py index 29cfff2c2d..6777ed1933 100644 --- a/ietf/group/utils.py +++ b/ietf/group/utils.py @@ -237,9 +237,10 @@ def construct_group_menu_context(request, group, selected, group_type, others): entries.append(("Review requests", urlreverse(ietf.group.views.review_requests, kwargs=kwargs))) entries.append(("Reviewers", urlreverse(ietf.group.views.reviewer_overview, kwargs=kwargs))) entries.append(("Reviews History", urlreverse(ietf.group.views.review_requests_history, kwargs=kwargs))) - if group.features.has_meetings: entries.append(("Meetings", urlreverse("ietf.group.views.meetings", kwargs=kwargs))) + if group.acronym in ["iesg"]: + entries.append(("Working Groups", urlreverse("ietf.iesg.views.working_groups"))) if group.acronym in ["iab", "iesg"]: entries.append(("Statements", urlreverse("ietf.group.views.statements", kwargs=kwargs))) entries.append(("Appeals", urlreverse("ietf.group.views.appeals", kwargs=kwargs))) @@ -250,7 +251,6 @@ def construct_group_menu_context(request, group, selected, group_type, others): if is_valid_url(group.list_archive): entries.append((mark_safe("List archive »"), group.list_archive)) - # actions actions = [] From d796aa0a9eee4b705bc072d546b9ce634eb7dd92 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 5 Nov 2025 15:29:08 -0500 Subject: [PATCH 002/158] chore: remove OBE test (#9885) --- ietf/liaisons/tests.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index 1209d7f401..d2f4fafca3 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -1119,17 +1119,6 @@ def test_redirect_for_approval(self): # ------------------------------------------------- # Form validations # ------------------------------------------------- - def test_post_and_send_fail(self): - RoleFactory(name_id='liaiman',person__user__username='ulm-liaiman',group__type_id='sdo',group__acronym='ulm') - GroupFactory(type_id='wg',acronym='mars') - - url = urlreverse('ietf.liaisons.views.liaison_add', kwargs={'type':'incoming'}) - login_testing_unauthorized(self, "ulm-liaiman", url) - - r = self.client.post(url,get_liaison_post_data(),follow=True) - - self.assertEqual(r.status_code, 200) - self.assertContains(r, 'As an IETF Liaison Manager you can not send incoming liaison statements') def test_deadline_field(self): '''Required for action, comment, not info, response''' From 2ac61062b4776933a3bb73554963095bf1503760 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 6 Nov 2025 10:33:38 -0500 Subject: [PATCH 003/158] feat: show available sdo groups to liaison coordinators and managers (#9856) * feat: show available sdo groups to liaison coordinators and mamagers * fix: better typing delcaration * fix: avoid unneeded whitespace in class declarations --- ietf/liaisons/tests.py | 11 +++ ietf/liaisons/urls.py | 1 + ietf/liaisons/views.py | 14 +++ ietf/templates/base/menu.html | 102 +++++++++++--------- ietf/templates/liaisons/list_other_sdo.html | 41 ++++++++ 5 files changed, 122 insertions(+), 47 deletions(-) create mode 100644 ietf/templates/liaisons/list_other_sdo.html diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index d2f4fafca3..5e0a237575 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -110,6 +110,17 @@ def test_help_pages(self): self.assertEqual(self.client.get('/liaison/help/from_ietf/').status_code, 200) self.assertEqual(self.client.get('/liaison/help/to_ietf/').status_code, 200) + def test_list_other_sdo(self): + GroupFactory(type_id="sdo", state_id="conclude", acronym="third") + GroupFactory(type_id="sdo", state_id="active", acronym="second") + GroupFactory(type_id="sdo", state_id="active", acronym="first") + url = urlreverse("ietf.liaisons.views.list_other_sdo") + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + q = PyQuery(r.content) + self.assertEqual(len(q("h1")), 2) + first_td_elements_text = [e.text for e in q("tr").find("td:first-child")] + self.assertEqual(first_td_elements_text, ["first", "second", "third"]) class UnitTests(TestCase): def test_get_contacts_for_liaison_messages_for_group_primary(self): diff --git a/ietf/liaisons/urls.py b/ietf/liaisons/urls.py index 0fbd29425e..498df3b965 100644 --- a/ietf/liaisons/urls.py +++ b/ietf/liaisons/urls.py @@ -37,4 +37,5 @@ url(r'^add/$', views.redirect_add), url(r'^for_approval/$', views.redirect_for_approval), url(r'^for_approval/(?P\d+)/$', views.redirect_for_approval), + url(r"^list_other_sdo/$", views.list_other_sdo), ] diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py index e2abff52ac..59c6ea69fc 100644 --- a/ietf/liaisons/views.py +++ b/ietf/liaisons/views.py @@ -509,3 +509,17 @@ def liaison_resend(request, object_id): messages.success(request,'Liaison Statement resent') return redirect('ietf.liaisons.views.liaison_list') + +@role_required("Secretariat", "IAB", "Liaison Coordinator", "Liaison Manager") +def list_other_sdo(request): + def _sdo_order_key(obj:Group)-> tuple[str,str]: + state_order = { + "active" : "a", + "conclude": "b", + } + return (state_order.get(obj.state.slug,f"c{obj.state.slug}"), obj.acronym) + + sdos = sorted(list(Group.objects.filter(type="sdo")),key = _sdo_order_key) + for sdo in sdos: + sdo.liaison_managers =[r.person for r in sdo.role_set.filter(name="liaiman")] + return render(request,"liaisons/list_other_sdo.html",dict(sdos=sdos)) diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html index 1e7c1688ff..8ff6e952da 100644 --- a/ietf/templates/base/menu.html +++ b/ietf/templates/base/menu.html @@ -31,19 +31,19 @@ {% if flavor == 'top' %}
    • {% endif %}
  • New work
  • - Chartering groups
  • - BOFs
  • - BOF Requests @@ -51,13 +51,13 @@ {% if flavor == 'top' %}
    • {% endif %}
  • Other groups
  • - Concluded groups
  • - Non-WG lists @@ -81,26 +81,26 @@
    • {% endif %}
  • - Search
  • - Recent I-Ds
  • - Submit an Internet-Draft
    • {% if user and user.is_authenticated %}
  • - My tracked docs @@ -108,7 +108,7 @@ {% if user|has_role:"Area Director,Secretariat" %} {% if flavor == 'top' %}
    • {% endif %}
  • - RFC status changes @@ -121,19 +121,19 @@
    • - IESG appeals
    • - IESG dashboard
    • - IESG statements @@ -142,7 +142,7 @@
    • - IRSG ballot status @@ -151,14 +151,14 @@ {% if flavor == 'top' %}
      • {% endif %}
    • Manage
    • - Approve an I-D
      • {% for g in user|docman_groups %}
    • - {{ g.acronym }} {{ g.type_id }} docs @@ -175,7 +175,7 @@
      • {% for g in user|managed_review_groups %}
    • - {{ g.acronym }} reviews @@ -192,7 +192,7 @@
      • {% for nomcom in nomcoms %}
    • - {{ nomcom|capfirst }} @@ -208,25 +208,25 @@ RFC streams
    • - IAB
    • - IRTF
    • - ISE
    • - Editorial @@ -239,15 +239,15 @@ Subseries
    • - STD - BCP - FYI @@ -273,37 +273,37 @@
      • {% endif %}
    • - Agenda
    • - Materials
    • - Floor plan
    • - Registration
    • - Important dates
    • - Request a session @@ -317,7 +317,7 @@
      • {% endif %}
    • - Session requests @@ -344,7 +344,7 @@
      • {% endif %}
    • - Upcoming meetings @@ -359,13 +359,13 @@
      • {% endif %}
    • - Past meetings
    • - Meeting proceedings @@ -391,56 +391,64 @@
      • {% endif %}
    • - IPR disclosures
    • - Liaison statements
      • + {% if user|has_role:"Secretariat,IAB,Liaison Manager,Liaison Coordinator" %}
    • - + List of other SDO groups + +
      • + {% endif %} +
    • + IESG agenda
    • - NomComs
    • - Downref registry
    • - Statistics
    • - API Help
    • - Release notes
    • - System status diff --git a/ietf/templates/liaisons/list_other_sdo.html b/ietf/templates/liaisons/list_other_sdo.html new file mode 100644 index 0000000000..23f2d94af3 --- /dev/null +++ b/ietf/templates/liaisons/list_other_sdo.html @@ -0,0 +1,41 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load origin static person_filters %} +{% block pagehead %} + +{% endblock %} +{% block title %} + Other SDO groups +{% endblock %} +{% block content %} + {% origin %} + {% regroup sdos by state.name as sdos_by_state %} + {% for sdos_in_state in sdos_by_state %} +

      {{sdos_in_state.grouper|capfirst}} SDO groups

      + + + + + + + + + + {% for group in sdos_in_state.list%} + + + + + + {% endfor %} + +
      AcronymNameLiaison Managers
      {{ group.acronym }}{{ group.name }} + {% for person in group.liaison_managers %} + {% person_link person %}{% if not forloop.last %}, {% endif %} + {% endfor %} +
      + {% endfor %} +{% endblock %} +{% block js %} + +{% endblock %} \ No newline at end of file From ce91feba9529ed74b8751269e70701b0ebeb70a4 Mon Sep 17 00:00:00 2001 From: Absit Iniuria Date: Fri, 7 Nov 2025 13:22:11 +0000 Subject: [PATCH 004/158] fix: onclick disable submit button (#9878) --- ietf/templates/submit/upload_submission.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ietf/templates/submit/upload_submission.html b/ietf/templates/submit/upload_submission.html index 7313d8f000..b8b1aca29c 100644 --- a/ietf/templates/submit/upload_submission.html +++ b/ietf/templates/submit/upload_submission.html @@ -73,6 +73,11 @@ $(document).ready(function() { if ($("#checkbox").is(':checked')) $("#other-formats").collapse('show') + + $("form").one('submit', function() { + $("button").attr('disabled', 'disabled'); + return true; + }) }); {% endblock %} From bd6a160508422da82ca24bdeabe08a9b7bcdb16f Mon Sep 17 00:00:00 2001 From: Russ Housley Date: Fri, 7 Nov 2025 08:24:11 -0500 Subject: [PATCH 005/158] fix: add most recent ballot hint to History tab. (#9851) * Add most recent ballot hint to History tab. Fixes#9696. * chore: style nit * fix: explicitly order queryset * fix: more precise description of the revision at the time of ballot * refactor: is None instead of == None --------- Co-authored-by: Robert Sparks Co-authored-by: Jennifer Richards --- ietf/doc/tests_utils.py | 28 +++++++++++++++++++++++- ietf/doc/utils.py | 19 +++++++++++++++- ietf/doc/views_doc.py | 6 +++++ ietf/templates/doc/document_history.html | 5 +++++ 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py index 7db59819da..ef71f6ae6e 100644 --- a/ietf/doc/tests_utils.py +++ b/ietf/doc/tests_utils.py @@ -13,13 +13,15 @@ from ietf.group.factories import GroupFactory, RoleFactory from ietf.name.models import DocTagName from ietf.person.factories import PersonFactory +from ietf.doc.factories import BallotPositionDocEventFactory from ietf.utils.test_utils import TestCase, name_of_file_containing, reload_db_objects from ietf.person.models import Person from ietf.doc.factories import DocumentFactory, WgRfcFactory, WgDraftFactory from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor from ietf.doc.utils import (update_action_holders, add_state_change_event, update_documentauthors, fuzzy_find_documents, rebuild_reference_relations, build_file_urls, - ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file) + ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file, + last_ballot_doc_revision) from ietf.utils.draft import Draft, PlaintextDraft from ietf.utils.xmldraft import XMLDraft @@ -533,3 +535,27 @@ def test_update_draft_bibxml_file(self, mock): self.assertEqual(mock.call_count, 1) self.assertEqual(mock.call_args, call(doc, "26")) self.assertEqual(ref_path.read_text(), "This\nis\nmy\nbibxml") + + +class LastBallotDocRevisionTests(TestCase): + def test_last_ballot_doc_revision(self): + now = timezone.now() + ad = Person.objects.get(user__username="ad") + bpde_with_null_send_email = BallotPositionDocEventFactory( + time=now - datetime.timedelta(minutes=30), + send_email=None, + ) + ballot = bpde_with_null_send_email.ballot + BallotPositionDocEventFactory( + ballot=ballot, + balloter=ad, + pos_id='noobj', + comment='Commentary', + comment_time=timezone.now(), + send_email=None, + ) + doc = bpde_with_null_send_email.doc + rev = bpde_with_null_send_email.rev + nobody = PersonFactory() + self.assertIsNone(last_ballot_doc_revision(doc, nobody)) + self.assertEqual(rev, last_ballot_doc_revision(doc, ad)) diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 115b28b09b..2bd9a3d314 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -36,11 +36,12 @@ from ietf.doc.models import Document, DocHistory, State, DocumentAuthor, DocHistoryAuthor from ietf.doc.models import RelatedDocument, RelatedDocHistory, BallotType, DocReminder from ietf.doc.models import DocEvent, ConsensusDocEvent, BallotDocEvent, IRSGBallotDocEvent, NewRevisionDocEvent, StateDocEvent -from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent +from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent, BallotPositionDocEvent from ietf.name.models import DocReminderTypeName, DocRelationshipName from ietf.group.models import Role, Group, GroupFeatures from ietf.ietfauth.utils import has_role, is_authorized_in_doc_stream, is_individual_draft_author, is_bofreq_editor from ietf.person.models import Email, Person +from ietf.person.utils import get_active_balloters from ietf.review.models import ReviewWish from ietf.utils import draft, log from ietf.utils.mail import parseaddr, send_mail @@ -687,6 +688,22 @@ def nice_consensus(consensus): } return mapping[consensus] +def last_ballot_doc_revision(doc, person): + """ Return the document revision for the most recent ballot position + by the provided user. """ + ballot = doc.active_ballot() + if ballot is None or person is None: + return None + balloters = get_active_balloters(ballot.ballot_type) + if person not in balloters: + return None + position_queryset = BallotPositionDocEvent.objects.filter(type="changed_ballot_position", balloter=person, ballot=ballot).order_by("-time") + if not position_queryset.exists(): + return None + ballot_time = position_queryset.first().time + doc_rev = NewRevisionDocEvent.objects.filter(doc=doc, time__lte=ballot_time).order_by('-time').first().rev + return doc_rev + def has_same_ballot(doc, date1, date2=None): """ Test if the most recent ballot created before the end of date1 is the same as the most recent ballot created before the diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 4a20db3c89..50070734f9 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -79,6 +79,7 @@ from ietf.doc.views_ballot import parse_ballot_edit_return_point from ietf.doc.forms import InvestigateForm, TelechatForm, NotifyForm, ActionHoldersForm, DocAuthorForm, DocAuthorChangeBasisForm from ietf.doc.mails import email_comment, email_remind_action_holders +from ietf.doc.utils import last_ballot_doc_revision from ietf.mailtrigger.utils import gather_relevant_expansions from ietf.meeting.models import Session, SessionPresentation from ietf.meeting.utils import group_sessions, get_upcoming_manageable_sessions, sort_sessions, add_event_info_to_session_qs @@ -1225,6 +1226,10 @@ def document_history(request, name): request.user, ("Area Director", "Secretariat", "IRTF Chair") ) + # if the current user has balloted on this document, give them a revision hint + ballot_doc_rev = None + if request.user.is_authenticated: + ballot_doc_rev = last_ballot_doc_revision(doc, request.user.person) return render( request, @@ -1235,6 +1240,7 @@ def document_history(request, name): "diff_revisions": diff_revisions, "events": events, "can_add_comment": can_add_comment, + "ballot_doc_rev": ballot_doc_rev, }, ) diff --git a/ietf/templates/doc/document_history.html b/ietf/templates/doc/document_history.html index a9ee3e9a36..78471e08d6 100644 --- a/ietf/templates/doc/document_history.html +++ b/ietf/templates/doc/document_history.html @@ -20,6 +20,11 @@

      Revision differences

      {% include "doc/document_history_form.html" with doc=doc diff_revisions=diff_revisions action=rfcdiff_base_url snapshot=snapshot only %} {% endif %} + {% if ballot_doc_rev %} +

      + Your most recent ballot position was provided when the document was at version {{ ballot_doc_rev }}. +

      + {% endif %}

      Document history

      {% if doc.came_from_draft %}
        From 1249ace05bc700c1b80e80aee30992eac0595e89 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Fri, 7 Nov 2025 14:32:08 -0500 Subject: [PATCH 006/158] feat: add OpenTelemetry integration (#9886) * feat: initial otel integration * feat: add traceparent propagation via headers and meta tag * feat: add memcached instrumentation * fix: add more otel service attributes * style: Update dev/build/gunicorn.conf.py Co-authored-by: Jennifer Richards * style: Update dev/build/gunicorn.conf.py Co-authored-by: Jennifer Richards * style: Update dev/build/gunicorn.conf.py Co-authored-by: Jennifer Richards * fix: Update traceparent meta tag to use otel variable * fix: Update traceparent_id context processor to use sub-object --------- Co-authored-by: Jennifer Richards --- dev/build/gunicorn.conf.py | 36 +++++++++++++++++++++++++++++++++++- ietf/context_processors.py | 6 ++++++ ietf/middleware.py | 10 ++++++++++ ietf/settings.py | 2 ++ ietf/templates/base.html | 3 ++- requirements.txt | 6 ++++++ 6 files changed, 61 insertions(+), 2 deletions(-) diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py index 032d95ee0d..c54b24a054 100644 --- a/dev/build/gunicorn.conf.py +++ b/dev/build/gunicorn.conf.py @@ -1,4 +1,16 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2025, All Rights Reserved + +import os +import ietf +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.instrumentation.django import DjangoInstrumentor +from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor +from opentelemetry.instrumentation.pymemcache import PymemcacheInstrumentor +from opentelemetry.instrumentation.requests import RequestsInstrumentor # Configure security scheme headers for forwarded requests. Cloudflare sets X-Forwarded-Proto # for us. Don't trust any of the other similar headers. Only trust the header if it's coming @@ -119,3 +131,25 @@ def post_request(worker, req, environ, resp): in_flight = in_flight_by_pid.get(worker.pid, []) if request_description in in_flight: in_flight.remove(request_description) + +def post_fork(server, worker): + server.log.info("Worker spawned (pid: %s)", worker.pid) + + resource = Resource.create(attributes={ + "service.name": "datatracker", + "service.version": ietf.__version__, + "service.instance.id": worker.pid, + "service.namespace": "datatracker", + "deployment.environment.name": os.environ.get("DATATRACKER_SERVICE_ENV", "dev") + }) + + trace.set_tracer_provider(TracerProvider(resource=resource)) + otlp_exporter = OTLPSpanExporter(endpoint="https://heimdall-otlp.ietf.org/v1/traces") + + trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter)) + + # Instrumentations + DjangoInstrumentor().instrument() + Psycopg2Instrumentor().instrument() + PymemcacheInstrumentor().instrument() + RequestsInstrumentor().instrument() diff --git a/ietf/context_processors.py b/ietf/context_processors.py index baa8d7a5d2..5aaa4ab256 100644 --- a/ietf/context_processors.py +++ b/ietf/context_processors.py @@ -5,6 +5,7 @@ from django.conf import settings from django.utils import timezone from ietf import __version__, __patch__, __release_branch__, __release_hash__ +from opentelemetry.propagate import inject def server_mode(request): return {'server_mode': settings.SERVER_MODE} @@ -51,3 +52,8 @@ def timezone_now(request): return { 'timezone_now': timezone.now(), } + +def traceparent_id(request): + context_extras = {} + inject(context_extras) + return { "otel": context_extras } diff --git a/ietf/middleware.py b/ietf/middleware.py index a4b7a0d24c..fa2e8efd0c 100644 --- a/ietf/middleware.py +++ b/ietf/middleware.py @@ -8,6 +8,7 @@ from django.http import HttpResponsePermanentRedirect from ietf.utils.log import log, exc_parts from ietf.utils.mail import log_smtp_exception +from opentelemetry.propagate import inject import re import smtplib import unicodedata @@ -99,3 +100,12 @@ def add_header(request): return response return add_header + +def add_otel_traceparent_header(get_response): + """Middleware to add the OpenTelemetry traceparent id header to the response""" + def add_header(request): + response = get_response(request) + inject(response) + return response + + return add_header diff --git a/ietf/settings.py b/ietf/settings.py index eb5f9d2161..f8d8a28d65 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -411,6 +411,7 @@ def skip_unreadable_post(record): ], 'OPTIONS': { 'context_processors': [ + 'ietf.context_processors.traceparent_id', 'django.contrib.auth.context_processors.auth', 'django.template.context_processors.debug', # makes 'sql_queries' available in templates 'django.template.context_processors.i18n', @@ -443,6 +444,7 @@ def skip_unreadable_post(record): MIDDLEWARE = [ + "ietf.middleware.add_otel_traceparent_header", "django.middleware.csrf.CsrfViewMiddleware", "corsheaders.middleware.CorsMiddleware", # see docs on CORS_REPLACE_HTTPS_REFERER before using it "django.middleware.common.CommonMiddleware", diff --git a/ietf/templates/base.html b/ietf/templates/base.html index d8ff85f86e..25ce50c467 100644 --- a/ietf/templates/base.html +++ b/ietf/templates/base.html @@ -15,6 +15,7 @@ {% block title %}No title{% endblock %} + @@ -170,4 +171,4 @@ {% analytical_body_bottom %} - \ No newline at end of file + diff --git a/requirements.txt b/requirements.txt index cf7c920fa3..02a4cf5fd0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -52,6 +52,12 @@ mock>=5.2.0 # should replace with unittest.mock and remove dependency types-mock>=5.2.0 mypy~=1.7.0 # Version requirements determined by django-stubs. oic>=1.7.0 # Used only by tests +opentelemetry-sdk>=1.38.0 +opentelemetry-instrumentation-django>=0.59b0 +opentelemetry-instrumentation-psycopg2>=0.59b0 +opentelemetry-instrumentation-pymemcache>=0.59b0 +opentelemetry-instrumentation-requests>=0.59b0 +opentelemetry-exporter-otlp-proto-http>=1.38.0 pillow>=11.3.0 psycopg2>=2.9.10 pyang>=2.6.1 From de08e5f0d2e9b862e2a642a7241f62c22be05d38 Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Fri, 7 Nov 2025 14:45:02 -0500 Subject: [PATCH 007/158] fix: Milestones date sorting out of order (#9868) * Add hidden YYYY-MM-DD field to sort * Also hide for ARIA readers --- ietf/templates/group/milestones.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ietf/templates/group/milestones.html b/ietf/templates/group/milestones.html index df03d370fd..51579ce80a 100644 --- a/ietf/templates/group/milestones.html +++ b/ietf/templates/group/milestones.html @@ -20,7 +20,7 @@

        - {% for group in sdos_in_state.list%} - + - + {% endif %} - {% if review_req.doc.authors %} + {% if review_req.doc.author_persons_or_names %} diff --git a/ietf/templates/group/manage_review_requests.html b/ietf/templates/group/manage_review_requests.html index 99b23c138a..d240ef24fa 100644 --- a/ietf/templates/group/manage_review_requests.html +++ b/ietf/templates/group/manage_review_requests.html @@ -66,10 +66,10 @@

        Auto-suggested
        {% endif %} - {% if r.doc.authors %} + {% if r.doc.author_persons_or_names %} Authors: - {% for person in r.doc.authors %} - {% person_link person %}{% if not forloop.last %},{% endif %} + {% for person, tp_name in r.doc.author_persons_or_names %} + {% if person %}{% person_link person %}{% else %}{{ tp_name }}{% endif %}{% if not forloop.last %},{% endif %} {% endfor %}
        {% endif %} diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py index 86c5a0c1c3..5faf83d93f 100644 --- a/ietf/utils/test_utils.py +++ b/ietf/utils/test_utils.py @@ -38,6 +38,7 @@ import re import email import html5lib +import rest_framework.test import requests_mock import shutil import sys @@ -312,3 +313,11 @@ def tearDown(self): shutil.rmtree(dir) self.requests_mock.stop() super().tearDown() + + +class APITestCase(TestCase): + """Test case that uses rest_framework's APIClient + + This is equivalent to rest_framework.test.APITestCase, but picks up our + """ + client_class = rest_framework.test.APIClient diff --git a/ietf/utils/validators.py b/ietf/utils/validators.py index 92a20f5a26..a99de72724 100644 --- a/ietf/utils/validators.py +++ b/ietf/utils/validators.py @@ -33,8 +33,9 @@ # Note that this is an instantiation of the regex validator, _not_ the # regex-string validator defined right below validate_no_control_chars = RegexValidator( - regex="^[^\x00-\x1f]*$", - message="Please enter a string without control characters." ) + regex="^[^\x01-\x1f]*$", + message="Please enter a string without control characters.", +) @deconstructible diff --git a/mypy.ini b/mypy.ini index 19df7ec9b0..4acaf98c95 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,6 +2,9 @@ ignore_missing_imports = True +# allow PEP 695 type aliases (flag needed until mypy >= 1.13) +enable_incomplete_feature = NewGenericSyntax + plugins = mypy_django_plugin.main diff --git a/requirements.txt b/requirements.txt index 02a4cf5fd0..3f89f6f16c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,8 +19,10 @@ django-analytical>=3.2.0 django-bootstrap5>=25.1 django-celery-beat>=2.7.0,<2.8.0 # pin until https://github.com/celery/django-celery-beat/issues/875 is resolved, then revisit django-celery-results>=2.6.0 +django-csp>=3.7 django-cors-headers>=4.7.0 django-debug-toolbar>=6.0.0 +django-filter>=24.3 django-markup>=1.10 # Limited use - need to reconcile against direct use of markdown django-oidc-provider==0.8.2 # 0.8.3 changes logout flow and claim return django-simple-history>=3.10.1 @@ -50,7 +52,7 @@ markdown>=3.8.0 types-markdown>=3.8.0 mock>=5.2.0 # should replace with unittest.mock and remove dependency types-mock>=5.2.0 -mypy~=1.7.0 # Version requirements determined by django-stubs. +mypy~=1.11.2 # Version requirements loosely determined by django-stubs. oic>=1.7.0 # Used only by tests opentelemetry-sdk>=1.38.0 opentelemetry-instrumentation-django>=0.59b0 From c845aa788b7d1b215636a7730756365c6d27ebd8 Mon Sep 17 00:00:00 2001 From: rjsparks <10996692+rjsparks@users.noreply.github.com> Date: Wed, 14 Jan 2026 18:11:50 +0000 Subject: [PATCH 042/158] ci: update base image target version to 20260114T1756 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index e05ad51d1b..41ff295eec 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20251219T2152 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260114T1756 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index f8afdadf36..3ad31c7e25 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20251219T2152 +20260114T1756 From c1e8eb658ce6a204f450cd6d61857d49f6bbfcee Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 15 Jan 2026 12:01:00 -0500 Subject: [PATCH 043/158] ci: Increase wait-for-completion timeouts in build.yml --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5e91445202..d97889fbb8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -426,7 +426,7 @@ jobs: token: ${{ secrets.GH_INFRA_K8S_TOKEN }} inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' wait-for-completion: true - wait-for-completion-timeout: 30m + wait-for-completion-timeout: 60m wait-for-completion-interval: 30s display-workflow-run-url: false @@ -453,7 +453,7 @@ jobs: token: ${{ secrets.GH_INFRA_K8S_TOKEN }} inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }' wait-for-completion: true - wait-for-completion-timeout: 60m + wait-for-completion-timeout: 120m wait-for-completion-interval: 20s display-workflow-run-url: false From ac2fced3113f16e54eb84fd2b5c1b3000f0f2c87 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 15 Jan 2026 14:58:04 -0400 Subject: [PATCH 044/158] fix: remove notice from bofreq template (#10265) --- ietf/templates/doc/bofreq/new_bofreq.html | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/ietf/templates/doc/bofreq/new_bofreq.html b/ietf/templates/doc/bofreq/new_bofreq.html index c6aa0054f9..cda6f73b90 100644 --- a/ietf/templates/doc/bofreq/new_bofreq.html +++ b/ietf/templates/doc/bofreq/new_bofreq.html @@ -1,5 +1,5 @@ {% extends "base.html" %} -{# Copyright The IETF Trust 2021, All Rights Reserved #} +{# Copyright The IETF Trust 2021-2026, All Rights Reserved #} {% load origin django_bootstrap5 static textfilters %} {% block title %}Start a new BOF Request{% endblock %} {% block content %} @@ -12,21 +12,6 @@

        Start a new BOF Request

      • Process: RFC 2418 Section 2.4
      • Considerations for having a successful BoF: RFC 5434
        • - {# The following block needs to be commented out after the BoF deadline and re-opened before next BoF request opening #} -
        -

        - Announcement for IETF 124: The IESG and the IAB have organized Ask Me Anything (AMA) virtual sessions - for the community to help proponents who are interested in putting up BoF proposals for IETF 124 - (see also the IETF-announce email): -

        -
          -
        • 28th of August 13:00-14:00 UTC -
          • -
        • 28th of August 19:00-20:00 UTC -
          • -
        -
        - {# End of the temporary block #}

        The IAB will also attempt to provide BoF Shepherds as described in their document on the subject only on request from the IESG. If you feel that your BoF would benefit from an IAB BoF Shepherd, please discuss this with your Area Director. From 0b6e887c36b37f3b66cf653360471f5151cd7796 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 16 Jan 2026 15:38:26 -0400 Subject: [PATCH 045/158] test: more robust URL coverage; improve badly formed test (#10270) * test: use bad ID with correct format * test: use Django resolver for URL coverage * chore: f-strings are hard --- ietf/api/tests_views_rpc.py | 5 +++- ietf/utils/test_runner.py | 56 ++++++++++++++++++++++++++++++------- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 032b4b9495..ece2af1b85 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -6,6 +6,7 @@ from django.conf import settings from django.core.files.base import ContentFile from django.db.models import Max +from django.db.models.functions import Coalesce from django.test.utils import override_settings from django.urls import reverse as urlreverse @@ -22,7 +23,9 @@ def test_draftviewset_references(self): viewname = "ietf.api.purple_api.draft-references" # non-existent draft - bad_id = Document.objects.aggregate(unused_id=Max("id") + 100)["unused_id"] + bad_id = Document.objects.aggregate(unused_id=Coalesce(Max("id"), 0) + 100)[ + "unused_id" + ] url = urlreverse(viewname, kwargs={"doc_id": bad_id}) # Without credentials r = self.client.get(url) diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 1a3d4e5c3d..df8ed1fd61 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -70,7 +70,7 @@ from django.template.loaders.filesystem import Loader as BaseLoader from django.test.runner import DiscoverRunner from django.core.management import call_command -from django.urls import URLResolver # type: ignore +from django.urls import URLResolver, resolve, Resolver404 # type: ignore from django.template.backends.django import DjangoTemplates from django.template.backends.django import Template # type: ignore[attr-defined] from django.utils import timezone @@ -88,6 +88,26 @@ from mypy_boto3_s3.service_resource import Bucket +class UrlCoverageWarning(UserWarning): + """Warning category for URL coverage-related warnings""" + pass + + +class UninterestingPatternWarning(UrlCoverageWarning): + """Warning category for unexpected URL match patterns + + These are common, caused by tests that hit a URL that is not selected for + coverage checking. The warning is in place to help with a putative future + review of whether we're selecting the right patterns to check for coverage. + """ + pass + + +# Configure warnings for reasonable output quantity +warnings.simplefilter("once", UrlCoverageWarning) +warnings.simplefilter("ignore", UninterestingPatternWarning) + + loaded_templates: set[str] = set() visited_urls: set[str] = set() test_database_name: Optional[str] = None @@ -550,21 +570,37 @@ def ignore_pattern(regex, pattern): ) or pattern.callback == django.views.static.serve) - patterns = [(regex, re.compile(regex, re.U), obj) for regex, obj in url_patterns - if not ignore_pattern(regex, obj)] + patterns ={ + regex: obj + for regex, obj in url_patterns + if not ignore_pattern(regex, obj) + } covered = set() for url in visited_urls: - for regex, compiled, obj in patterns: - if regex not in covered and compiled.match(url[1:]): # strip leading / - covered.add(regex) - break + try: + resolved = resolve(url) # let Django resolve the URL for us + except Resolver404: + warnings.warn( + f"Unable to resolve visited URL {url}", UrlCoverageWarning + ) + continue + if resolved.route not in patterns: + warnings.warn( + f"WARNING: url resolved to an unexpected pattern (url='{url}', " + f"resolved to r'{resolved.route}'", + UninterestingPatternWarning, + ) + continue + covered.add(resolved.route) self.runner.coverage_data["url"] = { - "coverage": 1.0*len(covered)/len(patterns), - "covered": dict( (k, (o.lookup_str, k in covered)) for k,p,o in patterns ), + "coverage": 1.0 * len(covered) / len(patterns), + "covered": dict( + (k, (o.lookup_str, k in covered)) for k, o in patterns.items() + ), "format": 4, - } + } self.report_test_result("url") else: From 0e9e18efa187857a8992aee867d659d3c0276d2f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 16 Jan 2026 16:21:49 -0400 Subject: [PATCH 046/158] chore: suppress expected warnings (#10272) --- ietf/utils/test_runner.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index df8ed1fd61..a23416e87f 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -90,7 +90,11 @@ class UrlCoverageWarning(UserWarning): """Warning category for URL coverage-related warnings""" - pass + # URLs for which we don't expect patterns to match + IGNORE_URLS = ( + "/_doesnotexist/", + "/sitemap.xml.", + ) class UninterestingPatternWarning(UrlCoverageWarning): @@ -581,9 +585,10 @@ def ignore_pattern(regex, pattern): try: resolved = resolve(url) # let Django resolve the URL for us except Resolver404: - warnings.warn( - f"Unable to resolve visited URL {url}", UrlCoverageWarning - ) + if url not in UrlCoverageWarning.IGNORE_URLS: + warnings.warn( + f"Unable to resolve visited URL {url}", UrlCoverageWarning + ) continue if resolved.route not in patterns: warnings.warn( From 337a2311cbef23d46e4dc290742215de764a701c Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 16 Jan 2026 14:28:12 -0600 Subject: [PATCH 047/158] feat: rsync rfc content, store in blob, rebuild references (#10255) * feat: rsync rfc content, store in blob, rebuild references * fix: isolate subprocess. Guard against missing file * fix: correct variable initialization. guard against unnecessary call * test: mock rsync task calls * fix: use list for typing rather than List * fix: string formatting * fix: generalize error string when there are no files to parse * fix: use delete_on_close with NamedTemporaryFile * fix: mtime is less distracting than m_time * fix: store the notprepped file on the fs * fix: typo * fix: fetch json, remove unneeded unlink * chore: ruff * fix: use list for typing * fix: typo * feat: bulk load rfcs into blob storage * fix: restrict the rsync_helper to rsync * test: test ietf.sync.utils * chore: honor typing choices * test: sync task tests * refactor: isolate the rsync from-file construction and test it * chore: ruff * fix: reflect current changes in older test * fix: address incorrect test assumption * chore: adhere to task naming conventions --- ietf/doc/tasks.py | 21 ++++++++++ ietf/doc/tests_utils.py | 4 +- ietf/doc/utils.py | 48 +++++++++++++++-------- ietf/settings.py | 1 + ietf/sync/tasks.py | 50 ++++++++++++++++++++++++ ietf/sync/tests.py | 82 +++++++++++++++++++++++++++++++++++++++- ietf/sync/tests_utils.py | 82 ++++++++++++++++++++++++++++++++++++++++ ietf/sync/utils.py | 69 +++++++++++++++++++++++++++++++++ 8 files changed, 338 insertions(+), 19 deletions(-) create mode 100644 ietf/sync/tests_utils.py create mode 100644 ietf/sync/utils.py diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index 4f7fe37782..02b7c2a07d 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -29,6 +29,7 @@ from .utils import ( generate_idnits2_rfc_status, generate_idnits2_rfcs_obsoleted, + rebuild_reference_relations, update_or_create_draft_bibxml_file, ensure_draft_bibxml_path_exists, investigate_fragment, @@ -128,3 +129,23 @@ def investigate_fragment_task(name_fragment: str): "name_fragment": name_fragment, "results": investigate_fragment(name_fragment), } + +@shared_task +def rebuild_reference_relations_task(doc_names: list[str]): + log.log(f"Task: Rebuilding reference relations for {doc_names}") + for doc in Document.objects.filter(name__in=doc_names, type__in=["rfc", "draft"]): + filenames = dict() + base = ( + settings.RFC_PATH + if doc.type_id == "rfc" + else settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR + ) + stem = doc.name if doc.type_id == "rfc" else f"{doc.name}-{doc.rev}" + for ext in ["xml", "txt"]: + path = Path(base) / f"{stem}.{ext}" + if path.is_file(): + filenames[ext] = str(path) + if len(filenames) > 0: + rebuild_reference_relations(doc, filenames) + else: + log.log(f"Found no content for {stem}") diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py index ef71f6ae6e..a2784bc85e 100644 --- a/ietf/doc/tests_utils.py +++ b/ietf/doc/tests_utils.py @@ -389,13 +389,13 @@ def test_requires_txt_or_xml(self): result = rebuild_reference_relations(self.doc, {}) self.assertCountEqual(result.keys(), ['errors']) self.assertEqual(len(result['errors']), 1) - self.assertIn('No Internet-Draft text available', result['errors'][0], + self.assertIn('No file available', result['errors'][0], 'Error should be reported if no Internet-Draft file is given') result = rebuild_reference_relations(self.doc, {'md': 'cant-do-this.md'}) self.assertCountEqual(result.keys(), ['errors']) self.assertEqual(len(result['errors']), 1) - self.assertIn('No Internet-Draft text available', result['errors'][0], + self.assertIn('No file available', result['errors'][0], 'Error should be reported if no XML or plaintext file is given') @patch.object(XMLDraft, 'get_refs') diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 0715471551..42fab7d472 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -941,50 +941,66 @@ def rebuild_reference_relations(doc, filenames): filenames should be a dict mapping file ext (i.e., type) to the full path of each file. """ - if doc.type.slug != 'draft': + if doc.type.slug not in ["draft", "rfc"]: return None + + log.log(f"Rebuilding reference relations for {doc.name}") # try XML first - if 'xml' in filenames: - refs = XMLDraft(filenames['xml']).get_refs() - elif 'txt' in filenames: - filename = filenames['txt'] + if "xml" in filenames: + refs = XMLDraft(filenames["xml"]).get_refs() + elif "txt" in filenames: + filename = filenames["txt"] try: refs = draft.PlaintextDraft.from_file(filename).get_refs() except IOError as e: - return { 'errors': ["%s :%s" % (e.strerror, filename)] } + return {"errors": [f"{e.strerror}: {filename}"]} else: - return {'errors': ['No Internet-Draft text available for rebuilding reference relations. Need XML or plaintext.']} + return { + "errors": [ + "No file available for rebuilding reference relations. Need XML or plaintext." + ] + } - doc.relateddocument_set.filter(relationship__slug__in=['refnorm','refinfo','refold','refunk']).delete() + doc.relateddocument_set.filter( + relationship__slug__in=["refnorm", "refinfo", "refold", "refunk"] + ).delete() warnings = [] errors = [] unfound = set() - for ( ref, refType ) in refs.items(): + for ref, refType in refs.items(): refdoc = Document.objects.filter(name=ref) if not refdoc and re.match(r"^draft-.*-\d{2}$", ref): refdoc = Document.objects.filter(name=ref[:-3]) count = refdoc.count() if count == 0: - unfound.add( "%s" % ref ) + unfound.add("%s" % ref) continue elif count > 1: - errors.append("Too many Document objects found for %s"%ref) + errors.append("Too many Document objects found for %s" % ref) else: # Don't add references to ourself if doc != refdoc[0]: - RelatedDocument.objects.get_or_create( source=doc, target=refdoc[ 0 ], relationship=DocRelationshipName.objects.get( slug='ref%s' % refType ) ) + RelatedDocument.objects.get_or_create( + source=doc, + target=refdoc[0], + relationship=DocRelationshipName.objects.get( + slug="ref%s" % refType + ), + ) if unfound: - warnings.append('There were %d references with no matching Document'%len(unfound)) + warnings.append( + "There were %d references with no matching Document" % len(unfound) + ) ret = {} if errors: - ret['errors']=errors + ret["errors"] = errors if warnings: - ret['warnings']=warnings + ret["warnings"] = warnings if unfound: - ret['unfound']=list(unfound) + ret["unfound"] = list(unfound) return ret diff --git a/ietf/settings.py b/ietf/settings.py index fedd313ca0..fd8d86a1ab 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -811,6 +811,7 @@ def skip_unreadable_post(record): "polls", "procmaterials", "review", + "rfc", "slides", "staging", "statchg", diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index e4174d3729..fc75a056ed 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -4,6 +4,8 @@ # import datetime import io +from pathlib import Path +from tempfile import NamedTemporaryFile import requests from celery import shared_task @@ -12,9 +14,11 @@ from django.utils import timezone from ietf.doc.models import DocEvent, RelatedDocument +from ietf.doc.tasks import rebuild_reference_relations_task from ietf.sync import iana from ietf.sync import rfceditor from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue +from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper from ietf.utils import log from ietf.utils.timezone import date_today @@ -65,11 +69,16 @@ def rfc_editor_index_update_task(full_index=False): if len(errata_data) < rfceditor.MIN_ERRATA_RESULTS: log.log("Not enough errata entries, only %s" % len(errata_data)) return # failed + newly_published = set() for rfc_number, changes, doc, rfc_published in rfceditor.update_docs_from_rfc_index( index_data, errata_data, skip_older_than_date=skip_date ): for c in changes: log.log("RFC%s, %s: %s" % (rfc_number, doc.name, c)) + if rfc_published: + newly_published.add(rfc_number) + if len(newly_published) > 0: + rsync_rfcs_from_rfceditor_task.delay(list(newly_published)) @shared_task @@ -222,3 +231,44 @@ def fix_subseries_docevents_task(): DocEvent.objects.filter(type="sync_from_rfc_editor", desc=desc).update( time=obsoleting_time ) + +@shared_task +def rsync_rfcs_from_rfceditor_task(rfc_numbers: list[int]): + log.log(f"Rsyncing rfcs from rfc-editor: {rfc_numbers}") + from_file = None + with NamedTemporaryFile(mode="w", delete_on_close=False) as fp: + fp.write(build_from_file_content(rfc_numbers)) + fp.close() + from_file = Path(fp.name) + rsync_helper( + [ + "-a", + "--ignore-existing", + f"--include-from={from_file}", + "--exclude=*", + "rsync.rfc-editor.org::rfcs/", + f"{settings.RFC_PATH}", + ] + ) + load_rfcs_into_blobdb(rfc_numbers) + + rebuild_reference_relations_task.delay([f"rfc{num}" for num in rfc_numbers]) + + +@shared_task +def load_rfcs_into_blobdb_task(start: int, end: int): + """Move file content for rfcs from rfc{start} to rfc{end} inclusive + + As this is expected to be removed once the blobdb is populated, it + will truncate its work to a coded max end. + This will not overwrite any existing blob content, and will only + log a small complaint if asked to load a non-exsiting RFC. + """ + # Protect us from ourselves + if end < start: + return + if start < 1: + start = 1 + if end > 11000: # Arbitrarily chosen + end = 11000 + load_rfcs_into_blobdb(list(range(start, end + 1))) diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 888920ae9d..bcc87a43aa 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -889,8 +889,9 @@ class TaskTests(TestCase): @mock.patch("ietf.sync.tasks.rfceditor.update_docs_from_rfc_index") @mock.patch("ietf.sync.tasks.rfceditor.parse_index") @mock.patch("ietf.sync.tasks.requests.get") + @mock.patch("ietf.sync.tasks.rsync_rfcs_from_rfceditor_task.delay") def test_rfc_editor_index_update_task( - self, requests_get_mock, parse_index_mock, update_docs_mock + self, rsync_task_mock, requests_get_mock, parse_index_mock, update_docs_mock ) -> None: # the annotation here prevents mypy from complaining about annotation-unchecked """rfc_editor_index_update_task calls helpers correctly @@ -922,6 +923,7 @@ def json(self): rfc = RfcFactory() # Test with full_index = False + rsync_task_mock.return_value = None requests_get_mock.side_effect = (index_response, errata_response) # will step through these parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) update_docs_mock.return_value = ( @@ -947,10 +949,13 @@ def json(self): ) self.assertIsNotNone(update_docs_kwargs["skip_older_than_date"]) + self.assertFalse(rsync_task_mock.called) + # Test again with full_index = True requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = (index_response, errata_response) # will step through these tasks.rfc_editor_index_update_task(full_index=True) @@ -971,40 +976,67 @@ def json(self): ) self.assertIsNone(update_docs_kwargs["skip_older_than_date"]) + self.assertFalse(rsync_task_mock.called) + + # Test again where the index would cause a new RFC to come into existance + requests_get_mock.reset_mock() + parse_index_mock.reset_mock() + update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() + requests_get_mock.side_effect = ( + index_response, + errata_response, + ) # will step through these + update_docs_mock.return_value = ( + (rfc.rfc_number, ("something changed",), rfc, True), + ) + tasks.rfc_editor_index_update_task(full_index=True) + self.assertTrue(rsync_task_mock.called) + rsync_task_args, rsync_task_kwargs = rsync_task_mock.call_args + self.assertEqual((([rfc.rfc_number],), {}), (rsync_task_args, rsync_task_kwargs)) + # Test error handling requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = requests.Timeout # timeout on every get() tasks.rfc_editor_index_update_task(full_index=False) self.assertFalse(parse_index_mock.called) self.assertFalse(update_docs_mock.called) + self.assertFalse(rsync_task_mock.called) requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = [index_response, requests.Timeout] # timeout second get() tasks.rfc_editor_index_update_task(full_index=False) self.assertFalse(update_docs_mock.called) + self.assertFalse(rsync_task_mock.called) requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = [index_response, errata_response] # feed in an index that is too short parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS - 1) tasks.rfc_editor_index_update_task(full_index=False) self.assertTrue(parse_index_mock.called) self.assertFalse(update_docs_mock.called) + self.assertFalse(rsync_task_mock.called) requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = [index_response, errata_response] errata_response.json_length = rfceditor.MIN_ERRATA_RESULTS - 1 # too short parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) tasks.rfc_editor_index_update_task(full_index=False) self.assertFalse(update_docs_mock.called) + self.assertFalse(rsync_task_mock.called) @override_settings(RFC_EDITOR_QUEUE_URL="https://rfc-editor.example.com/queue/") @mock.patch("ietf.sync.tasks.update_drafts_from_queue") @@ -1134,3 +1166,51 @@ def test_iana_protocols_update_task( self.assertTrue(requests_get_mock.called) self.assertFalse(parse_protocols_mock.called) self.assertFalse(update_rfc_log_mock.called) + + @mock.patch("ietf.sync.tasks.rsync_helper") + @mock.patch("ietf.sync.tasks.load_rfcs_into_blobdb") + @mock.patch("ietf.sync.tasks.rebuild_reference_relations_task.delay") + def test_rsync_rfcs_from_rfceditor_task( + self, + rebuild_relations_mock, + load_blobs_mock, + rsync_helper_mock, + ): + tasks.rsync_rfcs_from_rfceditor_task([12345, 54321]) + self.assertTrue(rsync_helper_mock.called) + self.assertTrue(load_blobs_mock.called) + load_blobs_args, load_blobs_kwargs = load_blobs_mock.call_args + self.assertEqual(load_blobs_args, ([12345, 54321],)) + self.assertEqual(load_blobs_kwargs, {}) + self.assertTrue(rebuild_relations_mock.called) + rebuild_args, rebuild_kwargs = rebuild_relations_mock.call_args + self.assertEqual(rebuild_args, (["rfc12345", "rfc54321"],)) + self.assertEqual(rebuild_kwargs, {}) + + @mock.patch("ietf.sync.tasks.load_rfcs_into_blobdb") + def test_load_rfcs_into_blobdb_task( + self, + load_blobs_mock, + ): + tasks.load_rfcs_into_blobdb_task(5, 3) + self.assertFalse(load_blobs_mock.called) + load_blobs_mock.reset_mock() + tasks.load_rfcs_into_blobdb_task(-1, 1) + self.assertTrue(load_blobs_mock.called) + mock_args, mock_kwargs = load_blobs_mock.call_args + self.assertEqual(mock_args, ([1],)) + self.assertEqual(mock_kwargs, {}) + load_blobs_mock.reset_mock() + tasks.load_rfcs_into_blobdb_task(10999, 50000) + self.assertTrue(load_blobs_mock.called) + mock_args, mock_kwargs = load_blobs_mock.call_args + self.assertEqual(mock_args, ([10999, 11000],)) + self.assertEqual(mock_kwargs, {}) + load_blobs_mock.reset_mock() + tasks.load_rfcs_into_blobdb_task(3261, 3263) + self.assertTrue(load_blobs_mock.called) + mock_args, mock_kwargs = load_blobs_mock.call_args + self.assertEqual(mock_args, ([3261, 3262, 3263],)) + self.assertEqual(mock_kwargs, {}) + + diff --git a/ietf/sync/tests_utils.py b/ietf/sync/tests_utils.py new file mode 100644 index 0000000000..eb4b4ddf74 --- /dev/null +++ b/ietf/sync/tests_utils.py @@ -0,0 +1,82 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from pathlib import Path +from tempfile import TemporaryDirectory + +from django.test import override_settings +from ietf import settings +from ietf.doc.storage_utils import exists_in_storage, retrieve_str +from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper +from ietf.utils.test_utils import TestCase + + +class RsyncHelperTests(TestCase): + def test_rsync_helper(self): + with ( + TemporaryDirectory() as source_dir, + TemporaryDirectory() as dest_dir, + ): + with (Path(source_dir) / "canary.txt").open("w") as canary_source_file: + canary_source_file.write("chirp") + rsync_helper( + [ + "-a", + f"{source_dir}/", + f"{dest_dir}/", + ] + ) + with (Path(dest_dir) / "canary.txt").open("r") as canary_dest_file: + chirp = canary_dest_file.read() + self.assertEqual(chirp, "chirp") + + def test_build_from_file_content(self): + content = build_from_file_content([12345, 54321]) + self.assertEqual( + content, + """prerelease/ +rfc12345.txt +rfc12345.html +rfc12345.xml +rfc12345.pdf +rfc12345.ps +rfc12345.json +prerelease/rfc12345.notprepped.xml +rfc54321.txt +rfc54321.html +rfc54321.xml +rfc54321.pdf +rfc54321.ps +rfc54321.json +prerelease/rfc54321.notprepped.xml +""", + ) + + +class RfcBlobUploadTests(TestCase): + def test_load_rfcs_into_blobdb(self): + with TemporaryDirectory() as faux_rfc_path: + with override_settings(RFC_PATH=faux_rfc_path): + rfc_path = Path(faux_rfc_path) + (rfc_path / "prerelease").mkdir() + for num in [12345, 54321]: + for ext in settings.RFC_FILE_TYPES + ("json",): + with (rfc_path / f"rfc{num}.{ext}").open("w") as f: + f.write(ext) + with (rfc_path / "rfc{num}.bogon").open("w") as f: + f.write("bogon") + with (rfc_path / "prerelease" / f"rfc{num}.notprepped.xml").open( + "w" + ) as f: + f.write("notprepped") + load_rfcs_into_blobdb([12345, 54321]) + for num in [12345, 54321]: + for ext in settings.RFC_FILE_TYPES + ("json",): + self.assertEqual( + retrieve_str("rfc", f"{ext}/rfc{num}.{ext}"), + ext, + ) + self.assertFalse(exists_in_storage("rfc", f"bogon/rfc{num}.bogon")) + self.assertEqual( + retrieve_str("rfc", f"notprepped/rfc{num}.notprepped.xml"), + "notprepped", + ) diff --git a/ietf/sync/utils.py b/ietf/sync/utils.py new file mode 100644 index 0000000000..5b5f8ff559 --- /dev/null +++ b/ietf/sync/utils.py @@ -0,0 +1,69 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime +import subprocess + +from pathlib import Path + +from django.conf import settings + +from ietf.utils import log +from ietf.doc.storage_utils import AlreadyExistsError, store_bytes + + +def rsync_helper(subprocess_arg_array: list[str]): + subprocess.run(["/usr/bin/rsync"]+subprocess_arg_array) + +def build_from_file_content(rfc_numbers: list[int]) -> str: + types_to_sync = settings.RFC_FILE_TYPES + ("json",) + lines = [] + lines.append("prerelease/") + for num in rfc_numbers: + for ext in types_to_sync: + lines.append(f"rfc{num}.{ext}") + lines.append(f"prerelease/rfc{num}.notprepped.xml") + return "\n".join(lines)+"\n" + +def load_rfcs_into_blobdb(numbers: list[int]): + types_to_load = settings.RFC_FILE_TYPES + ("json",) + for num in numbers: + for ext in types_to_load: + fs_path = Path(settings.RFC_PATH) / f"rfc{num}.{ext}" + if fs_path.is_file(): + with fs_path.open("rb") as f: + bytes = f.read() + mtime = fs_path.stat().st_mtime + try: + store_bytes( + kind="rfc", + name=f"{ext}/rfc{num}.{ext}", + content=bytes, + allow_overwrite=False, # Intentionally not allowing overwrite. + doc_name=f"rfc{num}", + doc_rev=None, + # Not setting content_type + mtime=datetime.datetime.fromtimestamp(mtime, tz=datetime.UTC), + ) + except AlreadyExistsError as e: + log.log(str(e)) + + # store the not-prepped xml + name = f"rfc{num}.notprepped.xml" + source = Path(settings.RFC_PATH) / "prerelease" / name + if source.is_file(): + with open(source, "rb") as f: + bytes = f.read() + mtime = source.stat().st_mtime + try: + store_bytes( + kind="rfc", + name=f"notprepped/{name}", + content=bytes, + allow_overwrite=False, # Intentionally not allowing overwrite. + doc_name=f"rfc{num}", + doc_rev=None, + # Not setting content_type + mtime=datetime.datetime.fromtimestamp(mtime, tz=datetime.UTC), + ) + except AlreadyExistsError as e: + log.log(str(e)) From 37888b36b3f7e2262b857363ff7b3b4486dbcbe2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 16 Jan 2026 16:28:53 -0400 Subject: [PATCH 048/158] feat: store blobs+set mtime in RFC publish API (#10260) * feat: set mtime for RFC pub files * chore: add rfc storage * refactor: destination helper is fs-specific * feat: RFC files->blobstore in publish API * test: test blob writing * chore: remove completed todo comment --- ietf/api/serializers_rpc.py | 7 +++++ ietf/api/tests_views_rpc.py | 58 +++++++++++++++++++++++++++++++++---- ietf/api/views_rpc.py | 57 ++++++++++++++++++++++++++++++++---- 3 files changed, 111 insertions(+), 11 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 2223f04aeb..f2e735be7a 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -5,6 +5,7 @@ from django.db import transaction from django.urls import reverse as urlreverse +from django.utils import timezone from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field from rest_framework import serializers @@ -571,6 +572,12 @@ class RfcFileSerializer(serializers.Serializer): "file types, but filenames are otherwise ignored." ), ) + mtime = serializers.DateTimeField( + required=False, + default=timezone.now, + default_timezone=datetime.UTC, + help_text="Modification timestamp to apply to uploaded files", + ) replace = serializers.BooleanField( required=False, default=False, diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index ece2af1b85..ecb50ee76c 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -10,6 +10,7 @@ from django.test.utils import override_settings from django.urls import reverse as urlreverse +from ietf.blobdb.models import Blob from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory from ietf.doc.models import RelatedDocument, Document from ietf.group.factories import RoleFactory, GroupFactory @@ -259,6 +260,31 @@ def _valid_post_data(): ) self.assertEqual(r.status_code, 400) + # Put a file in the way. Post should fail because replace = False + file_in_the_way = (rfc_path / f"rfc{unused_rfc_number}.txt") + file_in_the_way.touch() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + file_in_the_way.unlink() + + # Put a blob in the way. Post should fail because replace = False + blob_in_the_way = Blob.objects.create( + bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b"" + ) + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + blob_in_the_way.delete() + # valid post r = self.client.post( url, @@ -267,21 +293,41 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 200) - for suffix in [".xml", ".txt", ".html", ".pdf", ".json"]: + for extension in ["xml", "txt", "html", "pdf", "json"]: + filename = f"rfc{unused_rfc_number}.{extension}" self.assertEqual( - (rfc_path / f"rfc{unused_rfc_number}") - .with_suffix(suffix) + (rfc_path / filename) .read_text(), - f"This is {suffix}", - f"{suffix} file should contain the expected content", + f"This is .{extension}", + f"{extension} file should contain the expected content", + ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"{extension}/{filename}" + ).content + ), + f"This is .{extension}".encode("utf-8"), + f"{extension} blob should contain the expected content", ) + # special case for notprepped + notprepped_fn = f"rfc{unused_rfc_number}.notprepped.xml" self.assertEqual( ( - rfc_path / "prerelease" / f"rfc{unused_rfc_number}.notprepped.xml" + rfc_path / "prerelease" / notprepped_fn ).read_text(), "This is .notprepped.xml", ".notprepped.xml file should contain the expected content", ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"notprepped/{notprepped_fn}" + ).content + ), + b"This is .notprepped.xml", + ".notprepped.xml blob should contain the expected content", + ) # re-post with replace = False should now fail r = self.client.post( diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index fce174ab72..542836a857 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,4 +1,5 @@ # Copyright The IETF Trust 2023-2026, All Rights Reserved +import os import shutil from pathlib import Path from tempfile import TemporaryDirectory @@ -35,6 +36,7 @@ ) from ietf.doc.models import Document, DocHistory, RfcAuthor from ietf.doc.serializers import RfcAuthorSerializer +from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage from ietf.person.models import Email, Person @@ -366,8 +368,8 @@ class RfcPubFilesView(APIView): api_key_endpoint = "ietf.api.views_rpc" parser_classes = [parsers.MultiPartParser] - def _destination(self, filename: str | Path) -> Path: - """Destination for an uploaded RFC file + def _fs_destination(self, filename: str | Path) -> Path: + """Destination for an uploaded RFC file in the filesystem Strips any path components in filename and returns an absolute Path. """ @@ -378,6 +380,23 @@ def _destination(self, filename: str | Path) -> Path: return rfc_path / "prerelease" / filename.name return rfc_path / filename.name + def _blob_destination(self, filename: str | Path) -> str: + """Destination name for an uploaded RFC file in the blob store + + Strips any path components in filename and returns an absolute Path. + """ + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + file_type = "notprepped" + elif extension[0] == ".": + file_type = extension[1:] + else: + raise serializers.ValidationError( + f"Extension does not begin with '.'!? ({filename})", + ) + return f"{file_type}/{filename.name}" + @extend_schema( operation_id="upload_rfc_files", summary="Upload files for a published RFC", @@ -394,10 +413,17 @@ def post(self, request): uploaded_files = serializer.validated_data["contents"] # list[UploadedFile] replace = serializer.validated_data["replace"] dest_stem = f"rfc{rfc.rfc_number}" + mtime = serializer.validated_data["mtime"] + mtimestamp = mtime.timestamp() + blob_kind = "rfc" # List of files that might exist for an RFC possible_rfc_files = [ - self._destination(dest_stem + ext) + self._fs_destination(dest_stem + ext) + for ext in serializer.allowed_extensions + ] + possible_rfc_blobs = [ + self._blob_destination(dest_stem + ext) for ext in serializer.allowed_extensions ] if not replace: @@ -408,6 +434,14 @@ def post(self, request): "File(s) already exist for this RFC", code="files-exist", ) + for possible_existing_blob in possible_rfc_blobs: + if exists_in_storage( + kind=blob_kind, name=possible_existing_blob + ): + raise Conflict( + "Blob(s) already exist for this RFC", + code="blobs-exist", + ) with TemporaryDirectory() as tempdir: # Save files in a temporary directory. Use the uploaded filename @@ -421,14 +455,27 @@ def post(self, request): with tempfile_path.open("wb") as dest: for chunk in upfile.chunks(): dest.write(chunk) + os.utime(tempfile_path, (mtimestamp, mtimestamp)) files_to_move.append(tempfile_path) # copy files to final location, removing any existing ones first if the # remove flag was set if replace: for possible_existing_file in possible_rfc_files: possible_existing_file.unlink(missing_ok=True) + for possible_existing_blob in possible_rfc_blobs: + remove_from_storage( + blob_kind, possible_existing_blob, warn_if_missing=False + ) for ftm in files_to_move: - shutil.move(ftm, self._destination(ftm)) - # todo store in blob storage as well (need a bucket for RFCs) + with ftm.open("rb") as f: + store_file( + kind=blob_kind, + name=self._blob_destination(ftm), + file=f, + doc_name=rfc.name, + doc_rev=rfc.rev, # expect None, but match whatever it is + mtime=mtime, + ) + shutil.move(ftm, self._fs_destination(ftm)) return Response(NotificationAckSerializer().data) From 4aeb36e01803cf1fc93cc931aa53438417a75900 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 16 Jan 2026 16:46:48 -0400 Subject: [PATCH 049/158] chore: order is_auth'd middleware correctly (#10225) Needs to be earlier than ConditionalGetMiddleware or the flag will not be set for 304 responses. --- ietf/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/settings.py b/ietf/settings.py index fd8d86a1ab..1cda79e21b 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -454,6 +454,7 @@ def skip_unreadable_post(record): "django.contrib.sessions.middleware.SessionMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", + "ietf.middleware.is_authenticated_header_middleware", "django.middleware.http.ConditionalGetMiddleware", "simple_history.middleware.HistoryRequestMiddleware", # comment in this to get logging of SQL insert and update statements: @@ -464,7 +465,6 @@ def skip_unreadable_post(record): "django.middleware.clickjacking.XFrameOptionsMiddleware", "django.middleware.security.SecurityMiddleware", "ietf.middleware.unicode_nfkc_normalization_middleware", - "ietf.middleware.is_authenticated_header_middleware", ] ROOT_URLCONF = 'ietf.urls' From 3ff0154ce5044802c586961c8216e74393bbf6f6 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 16 Jan 2026 16:48:24 -0400 Subject: [PATCH 050/158] feat: include tickets in RegistrationResource (#10223) --- ietf/meeting/resources.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/ietf/meeting/resources.py b/ietf/meeting/resources.py index 88562a88fe..490b75f925 100644 --- a/ietf/meeting/resources.py +++ b/ietf/meeting/resources.py @@ -21,7 +21,13 @@ Attended, Registration, RegistrationTicket) -from ietf.name.resources import MeetingTypeNameResource +from ietf.name.resources import ( + AttendanceTypeNameResource, + MeetingTypeNameResource, + RegistrationTicketTypeNameResource, +) + + class MeetingResource(ModelResource): type = ToOneField(MeetingTypeNameResource, 'type') schedule = ToOneField('ietf.meeting.resources.ScheduleResource', 'schedule', null=True) @@ -437,11 +443,16 @@ class Meta: } api.meeting.register(AttendedResource()) -from ietf.meeting.resources import MeetingResource from ietf.person.resources import PersonResource class RegistrationResource(ModelResource): meeting = ToOneField(MeetingResource, 'meeting') person = ToOneField(PersonResource, 'person', null=True) + tickets = ToManyField( + 'ietf.meeting.resources.RegistrationTicketResource', + 'tickets', + full=True, + ) + class Meta: queryset = Registration.objects.all() serializer = api.Serializer() @@ -456,13 +467,17 @@ class Meta: "country_code": ALL, "email": ALL, "attended": ALL, + "checkedin": ALL, "meeting": ALL_WITH_RELATIONS, "person": ALL_WITH_RELATIONS, + "tickets": ALL_WITH_RELATIONS, } api.meeting.register(RegistrationResource()) class RegistrationTicketResource(ModelResource): registration = ToOneField(RegistrationResource, 'registration') + attendance_type = ToOneField(AttendanceTypeNameResource, 'attendance_type') + ticket_type = ToOneField(RegistrationTicketTypeNameResource, 'ticket_type') class Meta: queryset = RegistrationTicket.objects.all() serializer = api.Serializer() @@ -471,8 +486,8 @@ class Meta: ordering = ['id', ] filtering = { "id": ALL, - "ticket_type": ALL, - "attendance_type": ALL, + "ticket_type": ALL_WITH_RELATIONS, + "attendance_type": ALL_WITH_RELATIONS, "registration": ALL_WITH_RELATIONS, } api.meeting.register(RegistrationTicketResource()) From d53ceff5d408c0fc9daab4ebe7144e2507309fbf Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 21 Jan 2026 11:43:46 -0400 Subject: [PATCH 051/158] chore(dev): adjust dev settings for RFC file upload (#10287) * chore(dev): disable nginx max_body_size * chore(dev): create missing dest dir Create the "prerelease" directory, if needed, when accepting RFC publication files. Gated not to impact production, where this directory should already exist. --- docker/configs/nginx-proxy.conf | 1 + ietf/api/views_rpc.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/configs/nginx-proxy.conf b/docker/configs/nginx-proxy.conf index 3068cc71d7..5a9ae31ad0 100644 --- a/docker/configs/nginx-proxy.conf +++ b/docker/configs/nginx-proxy.conf @@ -4,6 +4,7 @@ server { proxy_read_timeout 1d; proxy_send_timeout 1d; + client_max_body_size 0; # disable checking root /var/www/html; index index.html index.htm index.nginx-debian.html; diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 542836a857..ea9c6348ca 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -476,6 +476,12 @@ def post(self, request): doc_rev=rfc.rev, # expect None, but match whatever it is mtime=mtime, ) - shutil.move(ftm, self._fs_destination(ftm)) + destination = self._fs_destination(ftm) + if ( + settings.SERVER_MODE != "production" + and not destination.parent.exists() + ): + destination.parent.mkdir() + shutil.move(ftm, destination) return Response(NotificationAckSerializer().data) From 50653e961556cf803b43707fad17c51077c68870 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 21 Jan 2026 11:44:49 -0400 Subject: [PATCH 052/158] fix: add Blob.__str__() (#10284) Keeps the model in sync with mailarchive's verison --- ietf/blobdb/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ietf/blobdb/models.py b/ietf/blobdb/models.py index fa7831f203..27325ada5d 100644 --- a/ietf/blobdb/models.py +++ b/ietf/blobdb/models.py @@ -64,6 +64,9 @@ class Meta: ), ] + def __str__(self): + return f"{self.bucket}:{self.name}" + def save(self, **kwargs): db = get_blobdb() with transaction.atomic(using=db): From 4ff48057e8dfc68d467effa396854dd7fdbede9a Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 21 Jan 2026 09:50:02 -0600 Subject: [PATCH 053/158] fix: restrict rfc bulkload to those with a Document object (#10289) * fix: restrict rfc bulkload to those with a Document object * fix: only query Document once * test: update test to match check for existing Document --- ietf/sync/tests_utils.py | 2 ++ ietf/sync/utils.py | 64 ++++++++++++++++++++++------------------ 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/ietf/sync/tests_utils.py b/ietf/sync/tests_utils.py index eb4b4ddf74..bb4a859e30 100644 --- a/ietf/sync/tests_utils.py +++ b/ietf/sync/tests_utils.py @@ -5,6 +5,7 @@ from django.test import override_settings from ietf import settings +from ietf.doc.factories import RfcFactory from ietf.doc.storage_utils import exists_in_storage, retrieve_str from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper from ietf.utils.test_utils import TestCase @@ -59,6 +60,7 @@ def test_load_rfcs_into_blobdb(self): rfc_path = Path(faux_rfc_path) (rfc_path / "prerelease").mkdir() for num in [12345, 54321]: + RfcFactory(rfc_number=num) for ext in settings.RFC_FILE_TYPES + ("json",): with (rfc_path / f"rfc{num}.{ext}").open("w") as f: f.write(ext) diff --git a/ietf/sync/utils.py b/ietf/sync/utils.py index 5b5f8ff559..b3bdd8d206 100644 --- a/ietf/sync/utils.py +++ b/ietf/sync/utils.py @@ -6,8 +6,8 @@ from pathlib import Path from django.conf import settings - from ietf.utils import log +from ietf.doc.models import Document from ietf.doc.storage_utils import AlreadyExistsError, store_bytes @@ -26,17 +26,42 @@ def build_from_file_content(rfc_numbers: list[int]) -> str: def load_rfcs_into_blobdb(numbers: list[int]): types_to_load = settings.RFC_FILE_TYPES + ("json",) + rfc_docs = Document.objects.filter(type="rfc", rfc_number__in=numbers).values_list("rfc_number", flat=True) for num in numbers: - for ext in types_to_load: - fs_path = Path(settings.RFC_PATH) / f"rfc{num}.{ext}" - if fs_path.is_file(): - with fs_path.open("rb") as f: + if num in rfc_docs: + for ext in types_to_load: + fs_path = Path(settings.RFC_PATH) / f"rfc{num}.{ext}" + if fs_path.is_file(): + with fs_path.open("rb") as f: + bytes = f.read() + mtime = fs_path.stat().st_mtime + try: + store_bytes( + kind="rfc", + name=f"{ext}/rfc{num}.{ext}", + content=bytes, + allow_overwrite=False, # Intentionally not allowing overwrite. + doc_name=f"rfc{num}", + doc_rev=None, + # Not setting content_type + mtime=datetime.datetime.fromtimestamp( + mtime, tz=datetime.UTC + ), + ) + except AlreadyExistsError as e: + log.log(str(e)) + + # store the not-prepped xml + name = f"rfc{num}.notprepped.xml" + source = Path(settings.RFC_PATH) / "prerelease" / name + if source.is_file(): + with open(source, "rb") as f: bytes = f.read() - mtime = fs_path.stat().st_mtime + mtime = source.stat().st_mtime try: store_bytes( kind="rfc", - name=f"{ext}/rfc{num}.{ext}", + name=f"notprepped/{name}", content=bytes, allow_overwrite=False, # Intentionally not allowing overwrite. doc_name=f"rfc{num}", @@ -46,24 +71,7 @@ def load_rfcs_into_blobdb(numbers: list[int]): ) except AlreadyExistsError as e: log.log(str(e)) - - # store the not-prepped xml - name = f"rfc{num}.notprepped.xml" - source = Path(settings.RFC_PATH) / "prerelease" / name - if source.is_file(): - with open(source, "rb") as f: - bytes = f.read() - mtime = source.stat().st_mtime - try: - store_bytes( - kind="rfc", - name=f"notprepped/{name}", - content=bytes, - allow_overwrite=False, # Intentionally not allowing overwrite. - doc_name=f"rfc{num}", - doc_rev=None, - # Not setting content_type - mtime=datetime.datetime.fromtimestamp(mtime, tz=datetime.UTC), - ) - except AlreadyExistsError as e: - log.log(str(e)) + else: + log.log( + f"Skipping loading rfc{num} into blobdb as no matching Document exists" + ) From a28594eecbe5000950ccd6a12bcadd5c61018183 Mon Sep 17 00:00:00 2001 From: Ryan Cross Date: Wed, 21 Jan 2026 09:55:37 -0800 Subject: [PATCH 054/158] feat: add option to email users about duplicate accounts. Fixes #8174. (#9850) * feat: add option to email users about duplicate accounts. Fixes #8174. * fix: split person merge into two views * fix: use form for validation * fix: update text of merge request email * fix: update copyright date * fix: use custom field classes in MergeRequestForm --- ietf/person/forms.py | 21 ++++- ietf/person/tests.py | 34 ++++++-- ietf/person/urls.py | 4 + ietf/person/views.py | 86 ++++++++++++++++--- ietf/templates/person/merge.html | 36 +------- ietf/templates/person/merge_request_email.txt | 23 +++++ ietf/templates/person/merge_submit.html | 57 ++++++++++++ ietf/templates/person/send_merge_request.html | 20 +++++ ietf/utils/fields.py | 23 ++++- 9 files changed, 254 insertions(+), 50 deletions(-) create mode 100644 ietf/templates/person/merge_request_email.txt create mode 100644 ietf/templates/person/merge_submit.html create mode 100644 ietf/templates/person/send_merge_request.html diff --git a/ietf/person/forms.py b/ietf/person/forms.py index 81ee362561..7eef8aa17b 100644 --- a/ietf/person/forms.py +++ b/ietf/person/forms.py @@ -1,15 +1,26 @@ -# Copyright The IETF Trust 2018-2020, All Rights Reserved +# Copyright The IETF Trust 2018-2025, All Rights Reserved # -*- coding: utf-8 -*- from django import forms + from ietf.person.models import Person +from ietf.utils.fields import MultiEmailField, NameAddrEmailField class MergeForm(forms.Form): source = forms.IntegerField(label='Source Person ID') target = forms.IntegerField(label='Target Person ID') + def __init__(self, *args, **kwargs): + self.readonly = False + if 'readonly' in kwargs: + self.readonly = kwargs.pop('readonly') + super().__init__(*args, **kwargs) + if self.readonly: + self.fields['source'].widget.attrs['readonly'] = True + self.fields['target'].widget.attrs['readonly'] = True + def clean_source(self): return self.get_person(self.cleaned_data['source']) @@ -21,3 +32,11 @@ def get_person(self, pk): return Person.objects.get(pk=pk) except Person.DoesNotExist: raise forms.ValidationError("ID does not exist") + + +class MergeRequestForm(forms.Form): + to = MultiEmailField() + frm = NameAddrEmailField() + reply_to = MultiEmailField() + subject = forms.CharField() + body = forms.CharField(widget=forms.Textarea) diff --git a/ietf/person/tests.py b/ietf/person/tests.py index 6326362fd8..f55d8b8a34 100644 --- a/ietf/person/tests.py +++ b/ietf/person/tests.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2014-2022, All Rights Reserved +# Copyright The IETF Trust 2014-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -10,7 +10,6 @@ from PIL import Image from pyquery import PyQuery - from django.core.exceptions import ValidationError from django.http import HttpRequest from django.test import override_settings @@ -23,6 +22,7 @@ from ietf.community.models import CommunityList from ietf.group.factories import RoleFactory from ietf.group.models import Group +from ietf.message.models import Message from ietf.nomcom.models import NomCom from ietf.nomcom.test_data import nomcom_test_data from ietf.nomcom.factories import NomComFactory, NomineeFactory, NominationFactory, FeedbackFactory, PositionFactory @@ -208,13 +208,13 @@ def test_merge(self): def test_merge_with_params(self): p1 = get_person_no_user() p2 = PersonFactory() - url = urlreverse("ietf.person.views.merge") + "?source={}&target={}".format(p1.pk, p2.pk) + url = urlreverse("ietf.person.views.merge_submit") + "?source={}&target={}".format(p1.pk, p2.pk) login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) self.assertContains(r, 'retaining login', status_code=200) def test_merge_with_params_bad_id(self): - url = urlreverse("ietf.person.views.merge") + "?source=1000&target=2000" + url = urlreverse("ietf.person.views.merge_submit") + "?source=1000&target=2000" login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) self.assertContains(r, 'ID does not exist', status_code=200) @@ -222,7 +222,7 @@ def test_merge_with_params_bad_id(self): def test_merge_post(self): p1 = get_person_no_user() p2 = PersonFactory() - url = urlreverse("ietf.person.views.merge") + url = urlreverse("ietf.person.views.merge_submit") expected_url = urlreverse("ietf.secr.rolodex.views.view", kwargs={'id': p2.pk}) login_testing_unauthorized(self, "secretary", url) data = {'source': p1.pk, 'target': p2.pk} @@ -451,6 +451,30 @@ def test_dots(self): ncchair = RoleFactory(group__acronym='nomcom2020',group__type_id='nomcom',name_id='chair').person self.assertEqual(get_dots(ncchair),['nomcom']) + def test_send_merge_request(self): + empty_outbox() + message_count_before = Message.objects.count() + source = PersonFactory() + target = PersonFactory() + url = urlreverse('ietf.person.views.send_merge_request') + url = url + f'?source={source.pk}&target={target.pk}' + login_testing_unauthorized(self, 'secretary', url) + r = self.client.get(url) + initial = r.context['form'].initial + subject = 'Action requested: Merging possible duplicate IETF Datatracker accounts' + self.assertEqual(initial['to'], ', '.join([source.user.username, target.user.username])) + self.assertEqual(initial['subject'], subject) + self.assertEqual(initial['reply_to'], 'support@ietf.org') + self.assertEqual(r.status_code, 200) + r = self.client.post(url, data=initial) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 1) + self.assertIn(source.user.username, outbox[0]['To']) + message_count_after = Message.objects.count() + message = Message.objects.last() + self.assertEqual(message_count_after, message_count_before + 1) + self.assertIn(source.user.username, message.to) + class TaskTests(TestCase): @mock.patch("ietf.person.tasks.log.log") diff --git a/ietf/person/urls.py b/ietf/person/urls.py index 867646fe39..f3eccd04b7 100644 --- a/ietf/person/urls.py +++ b/ietf/person/urls.py @@ -1,8 +1,12 @@ +# Copyright The IETF Trust 2009-2025, All Rights Reserved +# -*- coding: utf-8 -*- from ietf.person import views, ajax from ietf.utils.urls import url urlpatterns = [ url(r'^merge/?$', views.merge), + url(r'^merge/submit/?$', views.merge_submit), + url(r'^merge/send_request/?$', views.send_merge_request), url(r'^search/(?P(person|email))/$', views.ajax_select2_search), url(r'^(?P[0-9]+)/email.json$', ajax.person_email_json), url(r'^(?P[^/]+)$', views.profile), diff --git a/ietf/person/views.py b/ietf/person/views.py index a37b164311..d0b5912431 100644 --- a/ietf/person/views.py +++ b/ietf/person/views.py @@ -1,14 +1,16 @@ -# Copyright The IETF Trust 2012-2020, All Rights Reserved +# Copyright The IETF Trust 2012-2025, All Rights Reserved # -*- coding: utf-8 -*- from io import StringIO, BytesIO from PIL import Image +from django.conf import settings from django.contrib import messages from django.db.models import Q from django.http import HttpResponse, Http404 from django.shortcuts import render, redirect +from django.template.loader import render_to_string from django.utils import timezone import debug # pyflakes:ignore @@ -16,8 +18,9 @@ from ietf.ietfauth.utils import role_required from ietf.person.models import Email, Person from ietf.person.fields import select2_id_name_json -from ietf.person.forms import MergeForm +from ietf.person.forms import MergeForm, MergeRequestForm from ietf.person.utils import handle_users, merge_persons, lookup_persons +from ietf.utils.mail import send_mail_text def ajax_select2_search(request, model_name): @@ -98,16 +101,19 @@ def photo(request, email_or_name): @role_required("Secretariat") def merge(request): form = MergeForm() - method = 'get' + return render(request, 'person/merge.html', {'form': form}) + + +@role_required("Secretariat") +def merge_submit(request): change_details = '' warn_messages = [] source = None target = None if request.method == "GET": - form = MergeForm() if request.GET: - form = MergeForm(request.GET) + form = MergeForm(request.GET, readonly=True) if form.is_valid(): source = form.cleaned_data.get('source') target = form.cleaned_data.get('target') @@ -116,12 +122,9 @@ def merge(request): if source.user.last_login and target.user.last_login and source.user.last_login > target.user.last_login: warn_messages.append('WARNING: The most recently used login is being deleted!') change_details = handle_users(source, target, check_only=True) - method = 'post' - else: - method = 'get' if request.method == "POST": - form = MergeForm(request.POST) + form = MergeForm(request.POST, readonly=True) if form.is_valid(): source = form.cleaned_data.get('source') source_id = source.id @@ -136,11 +139,72 @@ def merge(request): messages.error(request, output) return redirect('ietf.secr.rolodex.views.view', id=target.pk) - return render(request, 'person/merge.html', { + return render(request, 'person/merge_submit.html', { 'form': form, - 'method': method, 'change_details': change_details, 'source': source, 'target': target, 'warn_messages': warn_messages, }) + + +@role_required("Secretariat") +def send_merge_request(request): + if request.method == 'GET': + merge_form = MergeForm(request.GET) + if merge_form.is_valid(): + source = merge_form.cleaned_data['source'] + target = merge_form.cleaned_data['target'] + to = [] + if source.email(): + to.append(source.email().address) + if target.email(): + to.append(target.email().address) + if source.user: + source_account = source.user.username + else: + source_account = source.email() + if target.user: + target_account = target.user.username + else: + target_account = target.email() + sender_name = request.user.person.name + subject = 'Action requested: Merging possible duplicate IETF Datatracker accounts' + context = { + 'source_account': source_account, + 'target_account': target_account, + 'sender_name': sender_name, + } + body = render_to_string('person/merge_request_email.txt', context) + initial = { + 'to': ', '.join(to), + 'frm': settings.DEFAULT_FROM_EMAIL, + 'reply_to': 'support@ietf.org', + 'subject': subject, + 'body': body, + 'by': request.user.person.pk, + } + form = MergeRequestForm(initial=initial) + else: + messages.error(request, "Error requesting merge email: " + merge_form.errors.as_text()) + return redirect("ietf.person.views.merge") + + if request.method == 'POST': + form = MergeRequestForm(request.POST) + if form.is_valid(): + extra = {"Reply-To": form.cleaned_data.get("reply_to")} + send_mail_text( + request, + form.cleaned_data.get("to"), + form.cleaned_data.get("frm"), + form.cleaned_data.get("subject"), + form.cleaned_data.get("body"), + extra=extra, + ) + + messages.success(request, "The merge confirmation email was sent.") + return redirect("ietf.person.views.merge") + + return render(request, "person/send_merge_request.html", { + "form": form, + }) diff --git a/ietf/templates/person/merge.html b/ietf/templates/person/merge.html index 36499ecdbc..5c3e6b0938 100644 --- a/ietf/templates/person/merge.html +++ b/ietf/templates/person/merge.html @@ -1,5 +1,5 @@ +{# Copyright The IETF Trust 2018-2025, All Rights Reserved #} {% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} {% load static %} {% load django_bootstrap5 %} {% block title %}Merge Persons{% endblock %} @@ -8,45 +8,17 @@

        Merge Person Records

        This tool will merge two Person records into one. If both records have logins and you want to retain the one on the left, use the Swap button to swap source and target records.

        -
        - {% if method == 'post' %} - {% csrf_token %} - {% endif %} +
        {% bootstrap_field form.source %} - {% if source %} - {% with person=source %} - {% include "person/person_info.html" %} - {% endwith %} - {% endif %}
        {% bootstrap_field form.target %} - {% if target %} - {% with person=target %} - {% include "person/person_info.html" %} - {% endwith %} - {% endif %}
        - {% if change_details %}{% endif %} - {% if warn_messages %} - {% for message in warn_messages %}{% endfor %} - {% endif %} - {% if method == 'post' %} - - Swap - - {% endif %} - {% endblock %} \ No newline at end of file diff --git a/ietf/templates/person/merge_request_email.txt b/ietf/templates/person/merge_request_email.txt new file mode 100644 index 0000000000..0a695f036c --- /dev/null +++ b/ietf/templates/person/merge_request_email.txt @@ -0,0 +1,23 @@ +Hello, + +We have identified multiple IETF Datatracker accounts that may represent a single person: + +https://datatracker.ietf.org/person/{{ source_account }} + +and + +https://datatracker.ietf.org/person/{{ target_account }} + +If this is so then it is important that we merge the accounts. + +This email is being sent to the primary emails associated with each Datatracker account. + +Please respond to this message individually from the email account(s) you control so we can take the appropriate action. + +If these should be merged, please identify which account you would like to keep the login credentials from. + +If you are associated with but no longer have access to one of the email accounts, then please let us know and we will follow up to determine how to proceed. + + +{{ sender_name }} +IETF Support \ No newline at end of file diff --git a/ietf/templates/person/merge_submit.html b/ietf/templates/person/merge_submit.html new file mode 100644 index 0000000000..30e1999f81 --- /dev/null +++ b/ietf/templates/person/merge_submit.html @@ -0,0 +1,57 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load django_bootstrap5 %} +{% block title %}Merge Persons{% endblock %} +{% block content %} +

        Merge Person Records

        +

        + This tool will merge two Person records into one. If both records have logins and you want to retain the one on the left, use the Swap button to swap source and target records. +

        +
        + {% csrf_token %} +
        +
        + {% bootstrap_field form.source %} + {% if source %} + {% with person=source %} + {% include "person/person_info.html" %} + {% endwith %} + {% endif %} +
        +
        + {% bootstrap_field form.target %} + {% if target %} + {% with person=target %} + {% include "person/person_info.html" %} + {% endwith %} + {% endif %} +
        +
        + {% if change_details %}{% endif %} + {% if warn_messages %} + {% for message in warn_messages %}{% endfor %} + {% endif %} + + + Swap + + + + + + Send Email + + + Back + + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/person/send_merge_request.html b/ietf/templates/person/send_merge_request.html new file mode 100644 index 0000000000..f0c6272dca --- /dev/null +++ b/ietf/templates/person/send_merge_request.html @@ -0,0 +1,20 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load django_bootstrap5 %} +{% block title %}Send Merge Notice{% endblock %} +{% block content %} +

        Send Merge Notice

        + {% if form.non_field_errors %}
        {{ form.non_field_errors }}
        {% endif %} +
        + {% csrf_token %} + {% bootstrap_field form.to layout='horizontal' %} + {% bootstrap_field form.frm layout='horizontal' %} + {% bootstrap_field form.reply_to layout='horizontal' %} + {% bootstrap_field form.subject layout='horizontal' %} + {% bootstrap_field form.body layout='horizontal' %} + + Cancel + +{% endblock %} diff --git a/ietf/utils/fields.py b/ietf/utils/fields.py index ba3fecebc6..6e8765612f 100644 --- a/ietf/utils/fields.py +++ b/ietf/utils/fields.py @@ -1,10 +1,11 @@ -# Copyright The IETF Trust 2012-2020, All Rights Reserved +# Copyright The IETF Trust 2012-2025, All Rights Reserved # -*- coding: utf-8 -*- import datetime import json import re +from email.utils import parseaddr import debug # pyflakes:ignore @@ -16,6 +17,7 @@ from django.core.exceptions import ValidationError from django.utils.dateparse import parse_duration + class MultiEmailField(forms.Field): def to_python(self, value): "Normalize data to a list of strings." @@ -38,6 +40,25 @@ def validate(self, value): for email in value: validate_email(email) + +def validate_name_addr_email(value): + "Validate name-addr style email address" + name, addr = parseaddr(value) + if not addr: + raise ValidationError("Invalid email format.") + try: + validate_email(addr) # validate the actual address part + except ValidationError: + raise ValidationError("Invalid email address.") + + +class NameAddrEmailField(forms.CharField): + def validate(self, value): + "Check if value consists only of valid emails." + super().validate(value) + validate_name_addr_email(value) + + def yyyymmdd_to_strftime_format(fmt): translation_table = sorted([ ("yyyy", "%Y"), From 5d2790bb3cfff4e45be076d56dcc0fb209ed327e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 22 Jan 2026 11:52:17 -0400 Subject: [PATCH 055/158] feat: additional fields for purple API (#10299) * feat: include group, abstract in purple draft API * feat: include ad/shepherd in purple API --- ietf/api/serializers_rpc.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index f2e735be7a..fe7f609251 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -114,11 +114,14 @@ class FullDraftSerializer(serializers.ModelSerializer): # is used for a writeable view, the validation will need to be added back. name = serializers.CharField(max_length=255) title = serializers.CharField(max_length=255) + group = serializers.SlugRelatedField(slug_field="acronym", read_only=True) # Other fields we need to add / adjust source_format = serializers.SerializerMethodField() authors = DocumentAuthorSerializer(many=True, source="documentauthor_set") - shepherd = serializers.SerializerMethodField() + shepherd = serializers.PrimaryKeyRelatedField( + source="shepherd.person", read_only=True + ) consensus = serializers.SerializerMethodField() class Meta: @@ -129,12 +132,16 @@ class Meta: "rev", "stream", "title", + "group", + "abstract", "pages", "source_format", "authors", "shepherd", "intended_std_level", "consensus", + "shepherd", + "ad", ] def get_consensus(self, doc: Document) -> Optional[bool]: @@ -155,12 +162,6 @@ def get_source_format( return "txt" return "unknown" - @extend_schema_field(OpenApiTypes.EMAIL) - def get_shepherd(self, doc: Document) -> str: - if doc.shepherd: - return doc.shepherd.formatted_ascii_email() - return "" - class DraftSerializer(FullDraftSerializer): class Meta: @@ -171,6 +172,7 @@ class Meta: "rev", "stream", "title", + "group", "pages", "source_format", "authors", From f56bfcb2f618fbaded92519797d4e4baa34ac4a1 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 23 Jan 2026 10:32:02 -0400 Subject: [PATCH 056/158] feat: more clickable message admin (#10307) --- ietf/message/admin.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ietf/message/admin.py b/ietf/message/admin.py index 250e1eb596..6a876cdc70 100644 --- a/ietf/message/admin.py +++ b/ietf/message/admin.py @@ -27,7 +27,8 @@ def queryset(self, request, queryset): class MessageAdmin(admin.ModelAdmin): - list_display = ["sent_status", "subject", "by", "time", "groups"] + list_display = ["sent_status", "display_subject", "by", "time", "groups"] + list_display_links = ["display_subject"] search_fields = ["subject", "body"] raw_id_fields = ["by", "related_groups", "related_docs"] list_filter = [ @@ -37,6 +38,10 @@ class MessageAdmin(admin.ModelAdmin): ordering = ["-time"] actions = ["retry_send"] + @admin.display(description="Subject", empty_value="(no subject)") + def display_subject(self, instance): + return instance.subject or None # None triggers the empty_value + def groups(self, instance): return ", ".join(g.acronym for g in instance.related_groups.all()) From 9c6fa92b7eed8b29bea96cb73148dba130678e63 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 27 Jan 2026 11:34:17 -0400 Subject: [PATCH 057/158] fix: update RPC publish API fields (#10308) * fix: update purple publish API fields * fix: handle IntegrityError more cleanly * fix: don't import RFC fields from draft * test: update test * chore: remove unused var/import * fix: f-string -> string --- ietf/api/serializers_rpc.py | 29 ++---------------------- ietf/api/tests_views_rpc.py | 44 ++++++++++++++++++------------------- ietf/api/views_rpc.py | 16 +++++++++++++- 3 files changed, 39 insertions(+), 50 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index fe7f609251..440c2a73d4 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -27,7 +27,7 @@ update_rfcauthors, ) from ietf.group.models import Group -from ietf.name.models import StreamName, StdLevelName, FormalLanguageName +from ietf.name.models import StreamName, StdLevelName from ietf.person.models import Person from ietf.utils import log @@ -137,7 +137,6 @@ class Meta: "pages", "source_format", "authors", - "shepherd", "intended_std_level", "consensus", "shepherd", @@ -263,15 +262,6 @@ class RfcPubSerializer(serializers.ModelSerializer): stream = serializers.PrimaryKeyRelatedField( queryset=StreamName.objects.filter(used=True) ) - formal_languages = serializers.PrimaryKeyRelatedField( - many=True, - required=False, - queryset=FormalLanguageName.objects.filter(used=True), - help_text=( - "formal languages used in RFC (defaults to those from draft, send empty" - "list to override)" - ) - ) std_level = serializers.PrimaryKeyRelatedField( queryset=StdLevelName.objects.filter(used=True), ) @@ -315,11 +305,8 @@ class Meta: "stream", "abstract", "pages", - "words", - "formal_languages", "std_level", "ad", - "note", "obsoletes", "updates", "subseries", @@ -353,9 +340,6 @@ def create(self, validated_data): # If specified, retrieve draft and extract RFC default values from it if draft_name is None: draft = None - defaults_from_draft = { - "group": Group.objects.get(acronym="none", type_id="individ"), - } else: # validation enforces that draft_name and draft_rev are both present draft = Document.objects.filter( @@ -378,17 +362,11 @@ def create(self, validated_data): }, code="already-published-draft", ) - defaults_from_draft = { - "ad": draft.ad, - "formal_languages": draft.formal_languages.all(), - "group": draft.group, - "note": draft.note, - } # Transaction to clean up if something fails with transaction.atomic(): # create rfc, letting validated request data override draft defaults - rfc = self._create_rfc(defaults_from_draft | validated_data) + rfc = self._create_rfc(validated_data) DocEvent.objects.create( doc=rfc, rev=rfc.rev, @@ -523,14 +501,11 @@ def create(self, validated_data): def _create_rfc(self, validated_data): authors_data = validated_data.pop("authors") - formal_languages = validated_data.pop("formal_languages", []) - # todo ad field rfc = Document.objects.create( type_id="rfc", name=f"rfc{validated_data['rfc_number']}", **validated_data, ) - rfc.formal_languages.set(formal_languages) # list of PKs is ok for order, author_data in enumerate(authors_data): rfc.rfcauthor_set.create( order=order, diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index ecb50ee76c..09fb40bf6e 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -80,9 +80,15 @@ def test_draftviewset_references(self): def test_notify_rfc_published(self): url = urlreverse("ietf.api.purple_api.notify_rfc_published") area = GroupFactory(type_id="area") + rfc_group = GroupFactory(type_id="wg") draft_ad = RoleFactory(group=area, name_id="ad").person - authors = PersonFactory.create_batch(2) - draft = WgDraftFactory(group__parent=area, authors=authors) + rfc_ad = PersonFactory() + draft_authors = PersonFactory.create_batch(2) + rfc_authors = PersonFactory.create_batch(3) + draft = WgDraftFactory( + group__parent=area, authors=draft_authors, ad=draft_ad, stream_id="ietf" + ) + rfc_stream_id = "ise" assert isinstance(draft, Document), "WgDraftFactory should generate a Document" unused_rfc_number = ( Document.objects.filter(rfc_number__isnull=False).aggregate( @@ -96,7 +102,7 @@ def test_notify_rfc_published(self): "draft_name": draft.name, "draft_rev": draft.rev, "rfc_number": unused_rfc_number, - "title": draft.title, + "title": "RFC " + draft.title, "authors": [ { "titlepage_name": f"titlepage {author.name}", @@ -106,17 +112,14 @@ def test_notify_rfc_published(self): "affiliation": "Some Affiliation", "country": "CA", } - for author in authors + for author in rfc_authors ], - "group": draft.group.acronym, - "stream": draft.stream_id, - "abstract": draft.abstract, - "pages": draft.pages, - "words": draft.pages * 250, - "formal_languages": [], + "group": rfc_group.acronym, + "stream": rfc_stream_id, + "abstract": "RFC version of " + draft.abstract, + "pages": draft.pages + 10, "std_level": "ps", - "ad": draft_ad.pk, - "note": "noted", + "ad": rfc_ad.pk, "obsoletes": [], "updates": [], "subseries": [], @@ -137,7 +140,7 @@ def test_notify_rfc_published(self): ).count(), 1, ) - self.assertEqual(rfc.title, draft.title) + self.assertEqual(rfc.title, "RFC " + draft.title) self.assertEqual(rfc.documentauthor_set.count(), 0) self.assertEqual( list( @@ -159,18 +162,15 @@ def test_notify_rfc_published(self): "affiliation": "Some Affiliation", "country": "CA", } - for author in authors + for author in rfc_authors ], ) - self.assertEqual(rfc.group, draft.group) - self.assertEqual(rfc.stream, draft.stream) - self.assertEqual(rfc.abstract, draft.abstract) - self.assertEqual(rfc.pages, draft.pages) - self.assertEqual(rfc.words, draft.pages * 250) - self.assertEqual(rfc.formal_languages.count(), 0) + self.assertEqual(rfc.group, rfc_group) + self.assertEqual(rfc.stream_id, rfc_stream_id) + self.assertEqual(rfc.abstract, "RFC version of " + draft.abstract) + self.assertEqual(rfc.pages, draft.pages + 10) self.assertEqual(rfc.std_level_id, "ps") - self.assertEqual(rfc.ad, draft_ad) - self.assertEqual(rfc.note, "noted") + self.assertEqual(rfc.ad, rfc_ad) self.assertEqual(rfc.related_that_doc("obs"), []) self.assertEqual(rfc.related_that_doc("updates"), []) self.assertEqual(rfc.part_of(), []) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index ea9c6348ca..6b1799f654 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -5,6 +5,7 @@ from tempfile import TemporaryDirectory from django.conf import settings +from django.db import IntegrityError from drf_spectacular.utils import OpenApiParameter from rest_framework import mixins, parsers, serializers, viewsets, status from rest_framework.decorators import action @@ -360,7 +361,20 @@ def post(self, request): serializer = RfcPubSerializer(data=request.data) serializer.is_valid(raise_exception=True) # Create RFC - serializer.save() + try: + serializer.save() + except IntegrityError as err: + if Document.objects.filter( + rfc_number=serializer.validated_data["rfc_number"] + ): + raise serializers.ValidationError( + "RFC with that number already exists", + code="rfc-number-in-use", + ) + raise serializers.ValidationError( + f"Unable to publish: {err}", + code="unknown-integrity-error", + ) return Response(NotificationAckSerializer().data) From 33fe0bcb7cafadc2096bfd1386c3d7e8a6a915f5 Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Wed, 28 Jan 2026 13:47:48 -0500 Subject: [PATCH 058/158] feat: add consensus in Draft serializer (#10327) --- ietf/api/serializers_rpc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 440c2a73d4..34e2c791c0 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -175,6 +175,7 @@ class Meta: "pages", "source_format", "authors", + "consensus", ] From a174f43574c5ed4f20ddcc0d600d03e5156fb351 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 30 Jan 2026 14:49:47 -0400 Subject: [PATCH 059/158] fix: use current time for bofreq revisions (#10333) * fix: use current time for bofreq revisions * test: test time handling Adjusts assertion argument order to match our usual style --- ietf/doc/tests_bofreq.py | 19 ++++++++++++------- ietf/doc/views_bofreq.py | 1 - 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ietf/doc/tests_bofreq.py b/ietf/doc/tests_bofreq.py index 6a7c9393ef..6b142149be 100644 --- a/ietf/doc/tests_bofreq.py +++ b/ietf/doc/tests_bofreq.py @@ -307,17 +307,20 @@ def test_submit(self): url = urlreverse('ietf.doc.views_bofreq.submit', kwargs=dict(name=doc.name)) rev = doc.rev + doc_time = doc.time r = self.client.post(url,{'bofreq_submission':'enter','bofreq_content':'# oiwefrase'}) self.assertEqual(r.status_code, 302) doc = reload_db_objects(doc) - self.assertEqual(rev, doc.rev) + self.assertEqual(doc.rev, rev) + self.assertEqual(doc.time, doc_time) nobody = PersonFactory() self.client.login(username=nobody.user.username, password=nobody.user.username+'+password') r = self.client.post(url,{'bofreq_submission':'enter','bofreq_content':'# oiwefrase'}) self.assertEqual(r.status_code, 403) doc = reload_db_objects(doc) - self.assertEqual(rev, doc.rev) + self.assertEqual(doc.rev, rev) + self.assertEqual(doc.time, doc_time) self.client.logout() editor = bofreq_editors(doc).first() @@ -339,12 +342,14 @@ def test_submit(self): r = self.client.post(url, postdict) self.assertEqual(r.status_code, 302) doc = reload_db_objects(doc) - self.assertEqual('%02d'%(int(rev)+1) ,doc.rev) - self.assertEqual(f'# {username}', doc.text()) - self.assertEqual(f'# {username}', retrieve_str('bofreq',doc.get_base_name())) - self.assertEqual(docevent_count+1, doc.docevent_set.count()) - self.assertEqual(1, len(outbox)) + self.assertEqual(doc.rev, '%02d'%(int(rev)+1)) + self.assertGreater(doc.time, doc_time) + self.assertEqual(doc.text(), f'# {username}') + self.assertEqual(retrieve_str('bofreq', doc.get_base_name()), f'# {username}') + self.assertEqual(doc.docevent_set.count(), docevent_count+1) + self.assertEqual(len(outbox), 1) rev = doc.rev + doc_time = doc.time finally: os.unlink(file.name) diff --git a/ietf/doc/views_bofreq.py b/ietf/doc/views_bofreq.py index 71cbe30491..94e3960dfa 100644 --- a/ietf/doc/views_bofreq.py +++ b/ietf/doc/views_bofreq.py @@ -91,7 +91,6 @@ def submit(request, name): by=request.user.person, rev=bofreq.rev, desc='New revision available', - time=bofreq.time, ) bofreq.save_with_history([e]) bofreq_submission = form.cleaned_data['bofreq_submission'] From 2dbe61e891752245447f4c2774353151f0bb34e5 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 9 Feb 2026 14:43:58 -0400 Subject: [PATCH 060/158] feat: speed up agenda.ics; cache more agenda data (#10362) * chore(dev): cprofile middleware settings * feat: precomputed agenda.ics (wip) * feat: precomp filtering support * fix: separately cache versioned hrefs Fixes https://github.com/ietf-tools/datatracker/issues/10355 * fix: versionless agenda href in agenda / ical * fix: preserve RESCHEDULED output * fix: fixup to agree with original output * feat: separate agenda cache, cache old meetings * feat: agenda refresh tasks * chore: undo accidental commit * chore: remove debug parameter * fix: convert session ID to int for comparison * test: update/fix tests, rename new task * refactor: rename task to have _task suffix Also changes a log msg so it won't contain "None" awkwardly * feat: no exceptions from agenda_data_refresh_task * test: explanatory comment * ci: agenda cache for k8s / testcrawl --- ietf/doc/models.py | 16 ++- ietf/meeting/tasks.py | 92 +++++++++++--- ietf/meeting/tests_tasks.py | 64 ++++++++-- ietf/meeting/views.py | 247 +++++++++++++++++++++++++++++++----- ietf/settings.py | 37 +++++- ietf/settings_testcrawl.py | 3 + k8s/settings_local.py | 10 ++ 7 files changed, 396 insertions(+), 73 deletions(-) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index cce9203d09..463aa6fd97 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -239,14 +239,14 @@ def revisions_by_newrevisionevent(self): return revisions def get_href(self, meeting=None): - return self._get_ref(meeting=meeting,meeting_doc_refs=settings.MEETING_DOC_HREFS) + return self._get_ref(meeting=meeting, versioned=True) def get_versionless_href(self, meeting=None): - return self._get_ref(meeting=meeting,meeting_doc_refs=settings.MEETING_DOC_GREFS) + return self._get_ref(meeting=meeting, versioned=False) - def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): + def _get_ref(self, meeting=None, versioned=True): """ Returns an url to the document text. This differs from .get_absolute_url(), which returns an url to the datatracker page for the document. @@ -255,12 +255,16 @@ def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): # the earlier resolution order, but there's at the moment one single # instance which matches this (with correct results), so we won't # break things all over the place. - if not hasattr(self, '_cached_href'): + cache_attr = "_cached_href" if versioned else "_cached_versionless_href" + if not hasattr(self, cache_attr): validator = URLValidator() if self.external_url and self.external_url.split(':')[0] in validator.schemes: validator(self.external_url) return self.external_url + meeting_doc_refs = ( + settings.MEETING_DOC_HREFS if versioned else settings.MEETING_DOC_GREFS + ) if self.type_id in settings.DOC_HREFS and self.type_id in meeting_doc_refs: if self.meeting_related(): self.is_meeting_related = True @@ -312,8 +316,8 @@ def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): if href.startswith('/'): href = settings.IDTRACKER_BASE_URL + href - self._cached_href = href - return self._cached_href + setattr(self, cache_attr, href) + return getattr(self, cache_attr) def set_state(self, state): """Switch state type implicit in state to state. This just diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py index c361325f9a..a73763560b 100644 --- a/ietf/meeting/tasks.py +++ b/ietf/meeting/tasks.py @@ -1,11 +1,14 @@ -# Copyright The IETF Trust 2024-2025, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # import datetime -from celery import shared_task -# from django.db.models import QuerySet +from itertools import batched + +from celery import shared_task, chain +from django.db.models import IntegerField +from django.db.models.functions import Cast from django.utils import timezone from ietf.utils import log @@ -19,9 +22,56 @@ from .utils import fetch_attendance_from_meetings +@shared_task +def agenda_data_refresh_task(num=None): + """Refresh agenda data for one plenary meeting + + If `num` is `None`, refreshes data for the current meeting. + """ + log.log( + f"Refreshing agenda data for {f"IETF-{num}" if num else "current IETF meeting"}" + ) + try: + generate_agenda_data(num, force_refresh=True) + except Exception as err: + # Log and swallow exceptions so failure on one meeting won't break a chain of + # tasks. This is used by agenda_data_refresh_all_task(). + log.log(f"ERROR: Refreshing agenda data failed for num={num}: {err}") + + @shared_task def agenda_data_refresh(): - generate_agenda_data(force_refresh=True) + """Deprecated. Use agenda_data_refresh_task() instead. + + TODO remove this after switching the periodic task to the new name + """ + log.log("Deprecated agenda_data_refresh task called!") + agenda_data_refresh_task() + + +@shared_task +def agenda_data_refresh_all_task(*, batch_size=10): + """Refresh agenda data for all plenary meetings + + Executes as a chain of tasks, each computing up to `batch_size` meetings + in a single task. + """ + meeting_numbers = sorted( + Meeting.objects.annotate( + number_as_int=Cast("number", output_field=IntegerField()) + ) + .filter(type_id="ietf", number_as_int__gt=64) + .values_list("number_as_int", flat=True) + ) + # Batch using chained maps rather than celery.chunk so we only use one worker + # at a time. + batched_task_chain = chain( + *( + agenda_data_refresh_task.map(nums) + for nums in batched(meeting_numbers, batch_size) + ) + ) + batched_task_chain.delay() @shared_task @@ -55,7 +105,9 @@ def proceedings_content_refresh_task(*, all=False): @shared_task def fetch_meeting_attendance_task(): # fetch most recent two meetings - meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by("-date")[:2] + meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by( + "-date" + )[:2] try: stats = fetch_attendance_from_meetings(meetings) except RuntimeError as err: @@ -64,8 +116,11 @@ def fetch_meeting_attendance_task(): for meeting, meeting_stats in zip(meetings, stats): log.log( "Fetched data for meeting {:>3}: {:4d} created, {:4d} updated, {:4d} deleted, {:4d} processed".format( - meeting.number, meeting_stats['created'], meeting_stats['updated'], meeting_stats['deleted'], - meeting_stats['processed'] + meeting.number, + meeting_stats["created"], + meeting_stats["updated"], + meeting_stats["deleted"], + meeting_stats["processed"], ) ) @@ -73,7 +128,7 @@ def fetch_meeting_attendance_task(): def _select_meetings( meetings: list[str] | None = None, meetings_since: str | None = None, - meetings_until: str | None = None + meetings_until: str | None = None, ): # nyah """Select meetings by number or date range""" # IETF-1 = 1986-01-16 @@ -130,15 +185,15 @@ def _select_meetings( @shared_task def resolve_meeting_materials_task( *, # only allow kw arguments - meetings: list[str] | None=None, - meetings_since: str | None=None, - meetings_until: str | None=None + meetings: list[str] | None = None, + meetings_since: str | None = None, + meetings_until: str | None = None, ): """Run materials resolver on meetings - + Can request a set of meetings by number by passing a list in the meetings arg, or by range by passing an iso-format timestamps in meetings_since / meetings_until. - To select all meetings, set meetings_since="zero" and omit other parameters. + To select all meetings, set meetings_since="zero" and omit other parameters. """ meetings_qs = _select_meetings(meetings, meetings_since, meetings_until) for meeting in meetings_qs.order_by("date"): @@ -155,7 +210,9 @@ def resolve_meeting_materials_task( f"meeting {meeting.number}: {err}" ) else: - log.log(f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds.") + log.log( + f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds." + ) @shared_task @@ -163,13 +220,13 @@ def store_meeting_materials_as_blobs_task( *, # only allow kw arguments meetings: list[str] | None = None, meetings_since: str | None = None, - meetings_until: str | None = None + meetings_until: str | None = None, ): """Push meeting materials into the blob store Can request a set of meetings by number by passing a list in the meetings arg, or by range by passing an iso-format timestamps in meetings_since / meetings_until. - To select all meetings, set meetings_since="zero" and omit other parameters. + To select all meetings, set meetings_since="zero" and omit other parameters. """ meetings_qs = _select_meetings(meetings, meetings_since, meetings_until) for meeting in meetings_qs.order_by("date"): @@ -187,4 +244,5 @@ def store_meeting_materials_as_blobs_task( ) else: log.log( - f"Blobs created in {(timezone.now() - mark).total_seconds():0.3f} seconds.") + f"Blobs created in {(timezone.now() - mark).total_seconds():0.3f} seconds." + ) diff --git a/ietf/meeting/tests_tasks.py b/ietf/meeting/tests_tasks.py index a5da00ecbf..2c5120a39d 100644 --- a/ietf/meeting/tests_tasks.py +++ b/ietf/meeting/tests_tasks.py @@ -5,23 +5,63 @@ from ietf.utils.test_utils import TestCase from ietf.utils.timezone import date_today from .factories import MeetingFactory -from .tasks import proceedings_content_refresh_task, agenda_data_refresh +from .tasks import ( + proceedings_content_refresh_task, + agenda_data_refresh_task, + agenda_data_refresh_all_task, +) from .tasks import fetch_meeting_attendance_task class TaskTests(TestCase): @patch("ietf.meeting.tasks.generate_agenda_data") - def test_agenda_data_refresh(self, mock_generate): - agenda_data_refresh() + def test_agenda_data_refresh_task(self, mock_generate): + agenda_data_refresh_task() self.assertTrue(mock_generate.called) - self.assertEqual(mock_generate.call_args, call(force_refresh=True)) + self.assertEqual(mock_generate.call_args, call(None, force_refresh=True)) + + mock_generate.reset_mock() + mock_generate.side_effect = RuntimeError + try: + agenda_data_refresh_task() + except Exception as err: + self.fail( + f"agenda_data_refresh_task should not raise exceptions (got {repr(err)})" + ) + + @patch("ietf.meeting.tasks.agenda_data_refresh_task") + @patch("ietf.meeting.tasks.chain") + def test_agenda_data_refresh_all_task(self, mock_chain, mock_agenda_data_refresh): + # Patch the agenda_data_refresh_task task with a mock whose `.map` attribute + # converts its argument, which is expected to be an iterator, to a list + # and returns it. We'll use this to check that the expected task chain + # was set up, but we don't actually run any celery tasks. + mock_agenda_data_refresh.map.side_effect = lambda x: list(x) + + meetings = MeetingFactory.create_batch(5, type_id="ietf") + numbers = sorted(int(m.number) for m in meetings) + agenda_data_refresh_all_task(batch_size=2) + self.assertTrue(mock_chain.called) + # The lists in the call() below are the output of the lambda we patched in + # via mock_agenda_data_refresh.map.side_effect above. I.e., this tests that + # map() was called with the correct batched data. + self.assertEqual( + mock_chain.call_args, + call( + [numbers[0], numbers[1]], + [numbers[2], numbers[3]], + [numbers[4]], + ), + ) + self.assertEqual(mock_agenda_data_refresh.call_count, 0) + self.assertEqual(mock_agenda_data_refresh.map.call_count, 3) @patch("ietf.meeting.tasks.generate_proceedings_content") def test_proceedings_content_refresh_task(self, mock_generate): # Generate a couple of meetings meeting120 = MeetingFactory(type_id="ietf", number="120") # 24 * 5 meeting127 = MeetingFactory(type_id="ietf", number="127") # 24 * 5 + 7 - + # Times to be returned now_utc = datetime.datetime.now(tz=datetime.UTC) hour_00_utc = now_utc.replace(hour=0) @@ -34,19 +74,19 @@ def test_proceedings_content_refresh_task(self, mock_generate): self.assertEqual(mock_generate.call_count, 1) self.assertEqual(mock_generate.call_args, call(meeting120, force_refresh=True)) mock_generate.reset_mock() - + # hour 01 - should call no meetings with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc): proceedings_content_refresh_task() self.assertEqual(mock_generate.call_count, 0) - + # hour 07 - should call meeting with number % 24 == 0 with patch("ietf.meeting.tasks.timezone.now", return_value=hour_07_utc): proceedings_content_refresh_task() self.assertEqual(mock_generate.call_count, 1) self.assertEqual(mock_generate.call_args, call(meeting127, force_refresh=True)) mock_generate.reset_mock() - + # With all=True, all should be called regardless of time. Reuse hour_01_utc which called none before with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc): proceedings_content_refresh_task(all=True) @@ -61,10 +101,10 @@ def test_fetch_meeting_attendance_task(self, mock_fetch_attendance): MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=3)), ] data = { - 'created': 1, - 'updated': 2, - 'deleted': 0, - 'processed': 3, + "created": 1, + "updated": 2, + "deleted": 0, + "processed": 3, } mock_fetch_attendance.return_value = [data, data] diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 903e3c7e79..8dccda9c87 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -40,7 +40,7 @@ from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile from django.core.validators import URLValidator -from django.urls import reverse,reverse_lazy +from django.urls import reverse, reverse_lazy, NoReverseMatch from django.db.models import F, Max, Q from django.forms.models import modelform_factory, inlineformset_factory from django.template import TemplateDoesNotExist @@ -1859,18 +1859,22 @@ def generate_agenda_data(num=None, force_refresh=False): :num: meeting number :force_refresh: True to force a refresh of the cache """ - cache = caches["default"] - cache_timeout = 6 * 60 - meeting = get_ietf_meeting(num) if meeting is None: raise Http404("No such full IETF meeting") elif int(meeting.number) <= 64: - return Http404("Pre-IETF 64 meetings are not available through this API") - else: - pass + raise Http404("Pre-IETF 64 meetings are not available through this API") + is_current_meeting = meeting.number == get_current_ietf_meeting_num() + + cache = caches["agenda"] + cache_timeout = ( + settings.AGENDA_CACHE_TIMEOUT_CURRENT_MEETING + if is_current_meeting + else settings.AGENDA_CACHE_TIMEOUT_DEFAULT + ) + cache_format = "1" # bump this on backward-incompatible data format changes - cache_key = f"generate_agenda_data_{meeting.number}" + cache_key = f"generate_agenda_data:{meeting.number}:v{cache_format}" if not force_refresh: cached_value = cache.get(cache_key) if cached_value is not None: @@ -1890,8 +1894,6 @@ def generate_agenda_data(num=None, force_refresh=False): filter_organizer = AgendaFilterOrganizer(assignments=filtered_assignments) - is_current_meeting = (num is None) or (num == get_current_ietf_meeting_num()) - # Get Floor Plans floors = FloorPlan.objects.filter(meeting=meeting).order_by('order') @@ -1966,21 +1968,32 @@ def api_get_session_materials(request, session_id=None): ) -def agenda_extract_schedule (item): +def agenda_extract_schedule(item): + if item.session.current_status == "resched": + resched_to = item.session.tombstone_for.official_timeslotassignment() + else: + resched_to = None return { "id": item.id, + "slug": item.slug(), "sessionId": item.session.id, - "room": item.room_name if item.timeslot.show_location else None, + "room": (item.timeslot.get_location() or None) if item.timeslot else None, "location": { "short": item.timeslot.location.floorplan.short, "name": item.timeslot.location.floorplan.name, } if (item.timeslot.show_location and item.timeslot.location and item.timeslot.location.floorplan) else {}, "acronym": item.acronym, - "duration": item.timeslot.duration.seconds, + "duration": item.timeslot.duration.total_seconds(), "name": item.session.name, + "slotId": item.timeslot.id, "slotName": item.timeslot.name, + "slotModified": item.timeslot.modified.isoformat(), "startDateTime": item.timeslot.time.isoformat(), "status": item.session.current_status, + "rescheduledTo": { + "startDateTime": resched_to.timeslot.time.isoformat(), + "duration": resched_to.timeslot.duration.total_seconds(), + } if resched_to is not None else {}, "type": item.session.type.slug, "purpose": item.session.purpose.slug, "isBoF": item.session.group_at_the_time().state_id == "bof", @@ -1998,7 +2011,7 @@ def agenda_extract_schedule (item): "showAgenda": True if (item.session.agenda() is not None or item.session.remote_instructions) else False }, "agenda": { - "url": item.session.agenda().get_href() + "url": item.session.agenda().get_versionless_href() } if item.session.agenda() is not None else { "url": None }, @@ -2290,10 +2303,131 @@ def ical_session_status(assignment): else: return "CONFIRMED" + +def render_icalendar_precomp(agenda_data): + ical_content = generate_agenda_ical_precomp(agenda_data) + return HttpResponse(ical_content, content_type="text/calendar") + + def render_icalendar(schedule, assignments): ical_content = generate_agenda_ical(schedule, assignments) return HttpResponse(ical_content, content_type="text/calendar") + +def generate_agenda_ical_precomp(agenda_data): + """Generate iCalendar from precomputed data using the icalendar library""" + + cal = Calendar() + cal.add("prodid", "-//IETF//datatracker.ietf.org ical agenda//EN") + cal.add("version", "2.0") + cal.add("method", "PUBLISH") + + meeting_data = agenda_data["meeting"] + for item in agenda_data["schedule"]: + event = Event() + + uid = f"ietf-{meeting_data["number"]}-{item["slotId"]}-{item["acronym"]}" + event.add("uid", uid) + + # add custom field with meeting's local TZ + event.add("x-meeting-tz", meeting_data["timezone"]) + + if item["name"]: + summary = item["name"] + else: + summary = f"{item["groupAcronym"]} - {item["groupName"]}" + + if item["note"]: + summary += f" ({item["note"]})" + + event.add("summary", summary) + + if item["room"]: + event.add("location", item["room"]) # room name + + if item["status"] == "canceled": + status = "CANCELLED" + elif item["status"] == "resched": + resched_to = item["rescheduledTo"] + if resched_to is None: + status = "RESCHEDULED" + else: + resched_start = datetime.datetime.fromisoformat( + resched_to["startDateTime"] + ) + dur = datetime.timedelta(seconds=resched_to["duration"]) + resched_end = resched_start + dur + formatted_start = resched_start.strftime("%A %H:%M").upper() + formatted_end = resched_end.strftime("%H:%M") + status = f"RESCHEDULED TO {formatted_start}-{formatted_end}" + else: + status = "CONFIRMED" + event.add("status", status) + + event.add("class", "PUBLIC") + + start_time = datetime.datetime.fromisoformat(item["startDateTime"]) + duration = datetime.timedelta(seconds=item["duration"]) + event.add("dtstart", start_time) + event.add("dtend", start_time + duration) + + # DTSTAMP: when the event was created or last modified (in UTC) + # n.b. timeslot.modified may not be an accurate measure of this + event.add("dtstamp", datetime.datetime.fromisoformat(item["slotModified"])) + + description_parts = [item["slotName"]] + + if item["note"]: + description_parts.append(f"Note: {item["note"]}") + + links = item["links"] + if links["onsiteTool"]: + description_parts.append(f"Onsite tool: {links["onsiteTool"]}") + + if links["videoStream"]: + description_parts.append(f"Meetecho: {links["videoStream"]}") + + if links["webex"]: + description_parts.append(f"Webex: {links["webex"]}") + + if item["remoteInstructions"]: + description_parts.append( + f"Remote instructions: {item["remoteInstructions"]}" + ) + + try: + materials_url = absurl( + "ietf.meeting.views.session_details", + num=meeting_data["number"], + acronym=item["acronym"], + ) + except NoReverseMatch: + pass + else: + description_parts.append(f"Session materials: {materials_url}") + event.add("url", materials_url) + + if meeting_data["number"].isdigit(): + try: + agenda_url = absurl("agenda", num=meeting_data["number"]) + except NoReverseMatch: + pass + else: + description_parts.append(f"See in schedule: {agenda_url}#row-{item["slug"]}") + + if item["agenda"] and item["agenda"]["url"]: + description_parts.append(f"Agenda {item["agenda"]["url"]}") + + # Join all description parts with 2 newlines + description = "\n\n".join(description_parts) + event.add("description", description) + + # Add event to calendar + cal.add_component(event) + + return cal.to_ical().decode("utf-8") + + def generate_agenda_ical(schedule, assignments): """Generate iCalendar using the icalendar library""" @@ -2428,10 +2562,66 @@ def parse_agenda_filter_params(querydict): def should_include_assignment(filter_params, assignment): """Decide whether to include an assignment""" - shown = len(set(filter_params['show']).intersection(assignment.filter_keywords)) > 0 - hidden = len(set(filter_params['hide']).intersection(assignment.filter_keywords)) > 0 + if hasattr(assignment, "filter_keywords"): + kw = assignment.filter_keywords + elif isinstance(assignment, dict): + kw = assignment.get("filterKeywords", []) + else: + raise ValueError("Unsupported assignment instance") + shown = len(set(filter_params['show']).intersection(kw)) > 0 + hidden = len(set(filter_params['hide']).intersection(kw)) > 0 return shown and not hidden + +def agenda_ical_ietf(meeting, filt_params, acronym=None, session_id=None): + agenda_data = generate_agenda_data(meeting.number, force_refresh=False) + if acronym: + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if item["groupAcronym"] == acronym + ] + elif session_id: + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if item["sessionId"] == session_id + ] + if filt_params is not None: + # Apply the filter + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if should_include_assignment(filt_params, item) + ] + return render_icalendar_precomp(agenda_data) + + +def agenda_ical_interim(meeting, filt_params, acronym=None, session_id=None): + schedule = get_schedule(meeting) + + if schedule is None and acronym is None and session_id is None: + raise Http404 + + assignments = SchedTimeSessAssignment.objects.filter( + schedule__in=[schedule, schedule.base], + session__on_agenda=True, + ) + assignments = preprocess_assignments_for_agenda(assignments, meeting) + AgendaKeywordTagger(assignments=assignments).apply() + + if filt_params is not None: + # Apply the filter + assignments = [a for a in assignments if should_include_assignment(filt_params, a)] + + if acronym: + assignments = [ a for a in assignments if a.session.group_at_the_time().acronym == acronym ] + elif session_id: + assignments = [ a for a in assignments if a.session_id == int(session_id) ] + + return render_icalendar(schedule, assignments) + + def agenda_ical(request, num=None, acronym=None, session_id=None): """Agenda ical view @@ -2459,33 +2649,20 @@ def agenda_ical(request, num=None, acronym=None, session_id=None): raise Http404 else: meeting = get_meeting(num, type_in=None) # get requested meeting, whatever its type - schedule = get_schedule(meeting) - if schedule is None and acronym is None and session_id is None: - raise Http404 - - assignments = SchedTimeSessAssignment.objects.filter( - schedule__in=[schedule, schedule.base], - session__on_agenda=True, - ) - assignments = preprocess_assignments_for_agenda(assignments, meeting) - AgendaKeywordTagger(assignments=assignments).apply() + if isinstance(session_id, str) and session_id.isdigit(): + session_id = int(session_id) try: filt_params = parse_agenda_filter_params(request.GET) except ValueError as e: return HttpResponseBadRequest(str(e)) - if filt_params is not None: - # Apply the filter - assignments = [a for a in assignments if should_include_assignment(filt_params, a)] - - if acronym: - assignments = [ a for a in assignments if a.session.group_at_the_time().acronym == acronym ] - elif session_id: - assignments = [ a for a in assignments if a.session_id == int(session_id) ] + if meeting.type_id == "ietf": + return agenda_ical_ietf(meeting, filt_params, acronym, session_id) + else: + return agenda_ical_interim(meeting, filt_params, acronym, session_id) - return render_icalendar(schedule, assignments) @cache_page(15 * 60) def agenda_json(request, num=None): diff --git a/ietf/settings.py b/ietf/settings.py index 1cda79e21b..899a377ad7 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -228,6 +228,10 @@ BLOBSTORAGE_CONNECT_TIMEOUT = 10 # seconds; boto3 default is 60 BLOBSTORAGE_READ_TIMEOUT = 10 # seconds; boto3 default is 60 +# Caching for agenda data in seconds +AGENDA_CACHE_TIMEOUT_DEFAULT = 8 * 24 * 60 * 60 # 8 days +AGENDA_CACHE_TIMEOUT_CURRENT_MEETING = 6 * 60 # 6 minutes + WSGI_APPLICATION = "ietf.wsgi.application" AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', ) @@ -1400,6 +1404,16 @@ def skip_unreadable_post(record): f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), }, + "agenda": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, "proceedings": { "BACKEND": "ietf.utils.cache.LenientMemcacheCache", "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", @@ -1453,6 +1467,17 @@ def skip_unreadable_post(record): "VERSION": __version__, "KEY_PREFIX": "ietf:dt", }, + "agenda": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + # "LOCATION": "127.0.0.1:11211", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, "proceedings": { "BACKEND": "django.core.cache.backends.dummy.DummyCache", # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", @@ -1519,11 +1544,17 @@ def skip_unreadable_post(record): NOMCOM_APP_SECRET = b'\x9b\xdas1\xec\xd5\xa0SI~\xcb\xd4\xf5t\x99\xc4i\xd7\x9f\x0b\xa9\xe8\xfeY\x80$\x1e\x12tN:\x84' ALLOWED_HOSTS = ['*',] - + try: # see https://github.com/omarish/django-cprofile-middleware - import django_cprofile_middleware # pyflakes:ignore - MIDDLEWARE = MIDDLEWARE + ['django_cprofile_middleware.middleware.ProfilerMiddleware', ] + import django_cprofile_middleware # pyflakes:ignore + + MIDDLEWARE = MIDDLEWARE + [ + "django_cprofile_middleware.middleware.ProfilerMiddleware", + ] + DJANGO_CPROFILE_MIDDLEWARE_REQUIRE_STAFF = ( + False # Do not use this setting for a public site! + ) except ImportError: pass diff --git a/ietf/settings_testcrawl.py b/ietf/settings_testcrawl.py index 40744a228d..edb978757a 100644 --- a/ietf/settings_testcrawl.py +++ b/ietf/settings_testcrawl.py @@ -27,6 +27,9 @@ 'MAX_ENTRIES': 10000, }, }, + 'agenda': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, 'proceedings': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', }, diff --git a/k8s/settings_local.py b/k8s/settings_local.py index f8ffacc83f..0386dbbdf9 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -306,6 +306,16 @@ def _multiline_to_list(s): f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), }, + "agenda": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, "proceedings": { "BACKEND": "ietf.utils.cache.LenientMemcacheCache", "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", From f8be1436fb9570eb623c6408cbb755e39bb18b22 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 10 Feb 2026 13:33:39 -0400 Subject: [PATCH 061/158] fix: add id attr to liaisons ButtonWidget (#10389) --- ietf/liaisons/widgets.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ietf/liaisons/widgets.py b/ietf/liaisons/widgets.py index 74368e83f2..48db8af0a3 100644 --- a/ietf/liaisons/widgets.py +++ b/ietf/liaisons/widgets.py @@ -26,7 +26,9 @@ def render(self, name, value, **kwargs): html += '%s' % conditional_escape(i) required_str = 'Please fill in %s to attach a new file' % conditional_escape(self.required_label) html += '%s' % conditional_escape(required_str) - html += '' % conditional_escape(self.label) + html += ''.format( + f"id_{name}", conditional_escape(self.label) + ) return mark_safe(html) From 4a024d9d64e36714523b2f9e04bdbc3005aefc03 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 10 Feb 2026 12:25:43 -0600 Subject: [PATCH 062/158] fix: identify editorial drafts that should not expire (#10388) --- ietf/doc/expire.py | 50 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py index bf8523aa98..d42af628f8 100644 --- a/ietf/doc/expire.py +++ b/ietf/doc/expire.py @@ -38,22 +38,46 @@ def expirable_drafts(queryset=None): # Populate this first time through (but after django has been set up) if nonexpirable_states is None: # all IESG states except I-D Exists and Dead block expiry - nonexpirable_states = list(State.objects.filter(used=True, type="draft-iesg").exclude(slug__in=("idexists", "dead"))) + nonexpirable_states = list( + State.objects.filter(used=True, type="draft-iesg").exclude( + slug__in=("idexists", "dead") + ) + ) # sent to RFC Editor and RFC Published block expiry (the latter # shouldn't be possible for an active draft, though) - nonexpirable_states += list(State.objects.filter(used=True, type__in=("draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"), slug__in=("rfc-edit", "pub"))) + nonexpirable_states += list( + State.objects.filter( + used=True, + type__in=( + "draft-stream-iab", + "draft-stream-irtf", + "draft-stream-ise", + "draft-stream-editorial", + ), + slug__in=("rfc-edit", "pub"), + ) + ) # other IRTF states that block expiration - nonexpirable_states += list(State.objects.filter(used=True, type_id="draft-stream-irtf", slug__in=("irsgpoll", "iesg-rev",))) - - return queryset.filter( - states__type="draft", states__slug="active" - ).exclude( - expires=None - ).exclude( - states__in=nonexpirable_states - ).exclude( - tags="rfc-rev" # under review by the RFC Editor blocks expiry - ).distinct() + nonexpirable_states += list( + State.objects.filter( + used=True, + type_id="draft-stream-irtf", + slug__in=( + "irsgpoll", + "iesg-rev", + ), + ) + ) + + return ( + queryset.filter(states__type="draft", states__slug="active") + .exclude(expires=None) + .exclude(states__in=nonexpirable_states) + .exclude( + tags="rfc-rev" # under review by the RFC Editor blocks expiry + ) + .distinct() + ) def get_soon_to_expire_drafts(days_of_warning): From 832c62e5c1f0cba736842cbe241236bfc8cf386f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 11 Feb 2026 11:41:58 -0400 Subject: [PATCH 063/158] feat: format+has_errata, drop see_also for red API (#10392) * fix: has_errata field for red API * chore: explanatory comment * feat: accurate format list for red API * refactor: specify blob names in API * chore: remove see_also field * fix: finish removing see_also field --- ietf/doc/api.py | 27 ++++++++++++++++++++++----- ietf/doc/models.py | 18 ++++++++++++++++++ ietf/doc/serializers.py | 33 +++++++++++++++++++++++---------- 3 files changed, 63 insertions(+), 15 deletions(-) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 47e7e6fffd..6a4c0c9fd5 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -1,7 +1,17 @@ # Copyright The IETF Trust 2024-2026, All Rights Reserved """Doc API implementations""" -from django.db.models import OuterRef, Subquery, Prefetch, Value, JSONField, QuerySet +from django.db.models import ( + BooleanField, + Count, + JSONField, + OuterRef, + Prefetch, + Q, + QuerySet, + Subquery, + Value, +) from django.db.models.functions import TruncDate from django_filters import rest_framework as filters from rest_framework import filters as drf_filters @@ -133,11 +143,18 @@ def augment_rfc_queryset(queryset: QuerySet[Document]): ) .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) .annotate( - # TODO implement these fake fields for real - see_also=Value([], output_field=JSONField()), - formats=Value(["txt", "xml"], output_field=JSONField()), + # Count of "verified-errata" tags will be 1 or 0, convert to Boolean + has_errata=Count( + "tags", + filter=Q( + tags__slug="verified-errata", + ), + output_field=BooleanField(), + ) + ) + .annotate( + # TODO implement this fake field for real keywords=Value(["keyword"], output_field=JSONField()), - errata=Value([], output_field=JSONField()), ) ) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 463aa6fd97..ec9a25add8 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1284,6 +1284,24 @@ def action_holders_enabled(self): iesg_state = self.get_state('draft-iesg') return iesg_state and iesg_state.slug != 'idexists' + def formats(self): + """List of file formats available + + Only implemented for RFCs. Relies on StoredObject. + """ + if self.type_id != "rfc": + raise RuntimeError("Only allowed for type=rfc") + return [ + { + "fmt": Path(object_name).parts[0], + "name": object_name, + } + for object_name in StoredObject.objects.filter( + store="rfc", doc_name=self.name, doc_rev=self.rev + ).values_list("name", flat=True) + ] + + class DocumentURL(models.Model): doc = ForeignKey(Document) tag = ForeignKey(DocUrlTagName) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index 05647d9ce1..e8d373164b 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -16,6 +16,7 @@ class RfcAuthorSerializer(serializers.ModelSerializer): """Serializer for an RfcAuthor / DocumentAuthor in a response""" + datatracker_person_path = serializers.URLField( source="person.get_absolute_url", required=False, @@ -36,7 +37,7 @@ class Meta: def to_representation(self, instance): """instance -> primitive data types - + Translates a DocumentAuthor into an equivalent RfcAuthor we can use the same serializer for either type. """ @@ -87,7 +88,15 @@ class DocIdentifierSerializer(serializers.Serializer): type RfcStatusSlugT = Literal[ - "std", "ps", "ds", "bcp", "inf", "exp", "hist", "unkn", "not-issued", + "std", + "ps", + "ds", + "bcp", + "inf", + "exp", + "hist", + "unkn", + "not-issued", ] @@ -188,11 +197,16 @@ class ContainingSubseriesSerializer(serializers.Serializer): type = serializers.CharField(source="source.type_id") +class RfcFormatSerializer(serializers.Serializer): + RFC_FORMATS = ("xml", "txt", "html", "pdf", "ps", "json", "notprepped") + + fmt = serializers.ChoiceField(choices=RFC_FORMATS) + name = serializers.CharField(help_text="Name of blob in the blob store") + + class RfcMetadataSerializer(serializers.ModelSerializer): """Serialize metadata of an RFC""" - RFC_FORMATS = ("xml", "txt", "html", "htmlized", "pdf", "ps") - number = serializers.IntegerField(source="rfc_number") published = serializers.DateField() status = RfcStatusSerializer(source="*") @@ -207,10 +221,11 @@ class RfcMetadataSerializer(serializers.ModelSerializer): updates = RelatedRfcSerializer(many=True, read_only=True) updated_by = ReverseRelatedRfcSerializer(many=True, read_only=True) subseries = ContainingSubseriesSerializer(many=True, read_only=True) - see_also = serializers.ListField(child=serializers.CharField(), read_only=True) - formats = serializers.MultipleChoiceField(choices=RFC_FORMATS) + formats = RfcFormatSerializer( + many=True, read_only=True, help_text="Available formats" + ) keywords = serializers.ListField(child=serializers.CharField(), read_only=True) - errata = serializers.ListField(child=serializers.CharField(), read_only=True) + has_errata = serializers.BooleanField(read_only=True) class Meta: model = Document @@ -230,15 +245,13 @@ class Meta: "updates", "updated_by", "subseries", - "see_also", "draft", "abstract", "formats", "keywords", - "errata", + "has_errata", ] - @extend_schema_field(RfcAuthorSerializer(many=True)) def get_authors(self, doc: Document): # If doc has any RfcAuthors, use those, otherwise fall back to DocumentAuthors From 66a1bf0a9ecae30036cf23fc07f6f26b27b1e94d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 11 Feb 2026 15:00:57 -0400 Subject: [PATCH 064/158] chore: un-squelch bibtexparser DeprecationWarnings (#10395) --- ietf/settings.py | 2 -- requirements.txt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index 899a377ad7..565e8825a9 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -36,8 +36,6 @@ warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="oic.utils.time_util") warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="pytz.tzinfo") warnings.filterwarnings("ignore", message="'instantiateVariableFont' is deprecated", module="weasyprint") -warnings.filterwarnings("ignore", category=DeprecationWarning, module="bibtexparser") # https://github.com/sciunto-org/python-bibtexparser/issues/502 -warnings.filterwarnings("ignore", category=DeprecationWarning, module="pyparsing") # https://github.com/sciunto-org/python-bibtexparser/issues/502 base_path = pathlib.Path(__file__).resolve().parent diff --git a/requirements.txt b/requirements.txt index 3f89f6f16c..cb583d5dc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ setuptools>=80.9.0 # Require this first, to prevent later errors aiosmtpd>=1.4.6 argon2-cffi>=25.1.0 # For the Argon2 password hasher option beautifulsoup4>=4.13.4 # Only used in tests -bibtexparser>=1.4.3 # Only used in tests +bibtexparser>=1.4.4 # Only used in tests bleach>=6.2.0 # project is deprecated but supported types-bleach>=6.2.0 boto3>=1.39.15 From 0b637ef4ace72f31a963b7603a8f69a253420810 Mon Sep 17 00:00:00 2001 From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com> Date: Wed, 11 Feb 2026 19:13:44 +0000 Subject: [PATCH 065/158] ci: update base image target version to 20260211T1901 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 41ff295eec..71370fabee 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260114T1756 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260211T1901 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 3ad31c7e25..947f3790e4 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260114T1756 +20260211T1901 From 492888b8a22113becc2cbe3900ba3294cfc6d7f6 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 12 Feb 2026 11:34:11 -0400 Subject: [PATCH 066/158] fix: handle doc_rev is None in Document.formats (#10401) Likely a temporary fix, but safer for quick deployment while we work on the bigger project. --- ietf/api/views_rpc.py | 2 +- ietf/doc/models.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 6b1799f654..2bf16480f2 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -487,7 +487,7 @@ def post(self, request): name=self._blob_destination(ftm), file=f, doc_name=rfc.name, - doc_rev=rfc.rev, # expect None, but match whatever it is + doc_rev=rfc.rev, # expect blank, but match whatever it is mtime=mtime, ) destination = self._fs_destination(ftm) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index ec9a25add8..8f700bf496 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -12,6 +12,8 @@ from io import BufferedReader from pathlib import Path + +from django.db.models import Q from lxml import etree from typing import Optional, Protocol, TYPE_CHECKING, Union from weasyprint import HTML as wpHTML @@ -1291,13 +1293,21 @@ def formats(self): """ if self.type_id != "rfc": raise RuntimeError("Only allowed for type=rfc") + + # StoredObject.doc_rev can be null or "" to represent no rev. Match either + # of these when self.rev is "" (always expected to be the case for RFCs) + rev_q = Q(doc_rev=self.rev) + if self.rev == "": + rev_q |= Q(doc_rev__isnull=True) return [ { "fmt": Path(object_name).parts[0], "name": object_name, } for object_name in StoredObject.objects.filter( - store="rfc", doc_name=self.name, doc_rev=self.rev + rev_q, + store="rfc", + doc_name=self.name, ).values_list("name", flat=True) ] From 1b306eb7f3b9b45b3162101c7fc216ac7da3eab1 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 12 Feb 2026 13:49:22 -0400 Subject: [PATCH 067/158] feat: bofreq timestamp fixup task (#10402) * feat: utility to fix up bofreq timestamps * fix: don't fix -00 (+ logging) * feat: task * chore: disable test coverage for one-off task --- ietf/doc/tasks.py | 8 ++- ietf/doc/utils_bofreq.py | 143 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 147 insertions(+), 4 deletions(-) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index 02b7c2a07d..b463b9cecf 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2024-2025, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # @@ -34,6 +34,7 @@ ensure_draft_bibxml_path_exists, investigate_fragment, ) +from .utils_bofreq import fixup_bofreq_timestamps @shared_task @@ -149,3 +150,8 @@ def rebuild_reference_relations_task(doc_names: list[str]): rebuild_reference_relations(doc, filenames) else: log.log(f"Found no content for {stem}") + + +@shared_task +def fixup_bofreq_timestamps_task(): # pragma: nocover + fixup_bofreq_timestamps() diff --git a/ietf/doc/utils_bofreq.py b/ietf/doc/utils_bofreq.py index aec8f60ad6..d01b039b8e 100644 --- a/ietf/doc/utils_bofreq.py +++ b/ietf/doc/utils_bofreq.py @@ -1,12 +1,149 @@ -# Copyright The IETF Trust 2021 All Rights Reserved +# Copyright The IETF Trust 2021-2026 All Rights Reserved +import datetime +from pathlib import Path -from ietf.doc.models import BofreqEditorDocEvent, BofreqResponsibleDocEvent +from django.conf import settings + +from ietf.doc.models import ( + BofreqEditorDocEvent, + BofreqResponsibleDocEvent, + DocEvent, + DocHistory, + Document, +) from ietf.person.models import Person +from ietf.utils import log + def bofreq_editors(bofreq): e = bofreq.latest_event(BofreqEditorDocEvent) return e.editors.all() if e else Person.objects.none() + def bofreq_responsible(bofreq): e = bofreq.latest_event(BofreqResponsibleDocEvent) - return e.responsible.all() if e else Person.objects.none() \ No newline at end of file + return e.responsible.all() if e else Person.objects.none() + + +def fixup_bofreq_timestamps(): # pragma: nocover + """Fixes bofreq event / document timestamps + + Timestamp errors resulted from the bug fixed by + https://github.com/ietf-tools/datatracker/pull/10333 + + Does not fix up -00 revs because the timestamps on these were not affected by + the bug. Replacing their timestamps creates a confusing event history because the + filesystem timestamp is usually a fraction of a second later than other events + created upon the initial rev creation. This causes the "New revision available" + event to appear _after_ these events in the history. Better to leave them as is. + """ + FIX_DEPLOYMENT_TIME = "2026-02-03T01:16:00+00:00" # 12.58.0 -> production + + def _get_doc_time(doc_name: str, rev: str): + path = Path(settings.BOFREQ_PATH) / f"{doc_name}-{rev}.md" + return datetime.datetime.fromtimestamp(path.stat().st_mtime, datetime.UTC) + + # Find affected DocEvents and DocHistories + new_bofreq_events = ( + DocEvent.objects.filter( + doc__type="bofreq", type="new_revision", time__lt=FIX_DEPLOYMENT_TIME + ) + .exclude(rev="00") # bug did not affect rev 00 events + .order_by("doc__name", "rev") + ) + log.log( + f"fixup_bofreq_timestamps: found {new_bofreq_events.count()} " + f"new_revision events before {FIX_DEPLOYMENT_TIME}" + ) + document_fixups = {} + for e in new_bofreq_events: + name = e.doc.name + rev = e.rev + filesystem_time = _get_doc_time(name, rev) + assert e.time < filesystem_time, ( + f"Rev {rev} event timestamp for {name} unexpectedly later than the " + "filesystem timestamp!" + ) + try: + dochistory = DocHistory.objects.filter( + name=name, time__lt=filesystem_time + ).get(rev=rev) + except DocHistory.MultipleObjectsReturned as err: + raise RuntimeError( + f"Multiple DocHistories for {name} rev {rev} exist earlier than the " + "filesystem timestamp!" + ) from err + except DocHistory.DoesNotExist as err: + if rev == "00": + # Unreachable because we don't adjust -00 revs, but could be needed + # if we did, in theory. In practice it's still not reached, but + # keeping the case for completeness. + dochistory = None + else: + raise RuntimeError( + f"No DocHistory for {name} rev {rev} exists earlier than the " + f"filesystem timestamp!" + ) from err + + if name not in document_fixups: + document_fixups[name] = [] + document_fixups[name].append( + { + "event": e, + "dochistory": dochistory, + "filesystem_time": filesystem_time, + } + ) + + # Now do the actual fixup + system_person = Person.objects.get(name="(System)") + for doc_name, fixups in document_fixups.items(): + bofreq = Document.objects.get(type="bofreq", name=doc_name) + log_msg_parts = [] + adjusted_revs = [] + for fixup in fixups: + event_to_fix = fixup["event"] + dh_to_fix = fixup["dochistory"] + new_time = fixup["filesystem_time"] + adjusted_revs.append(event_to_fix.rev) + + # Fix up the event + event_to_fix.time = new_time + event_to_fix.save() + log_msg_parts.append(f"rev {event_to_fix.rev} DocEvent") + + # Fix up the DocHistory + if dh_to_fix is not None: + dh_to_fix.time = new_time + dh_to_fix.save() + log_msg_parts.append(f"rev {dh_to_fix.rev} DocHistory") + + if event_to_fix.rev == bofreq.rev and bofreq.time < new_time: + # Update the Document without calling save(). Only update if + # the time has not changed so we don't inadvertently overwrite + # a concurrent update. + Document.objects.filter(pk=bofreq.pk, time=bofreq.time).update( + time=new_time + ) + bofreq.refresh_from_db() + if bofreq.rev == event_to_fix.rev: + log_msg_parts.append(f"rev {bofreq.rev} Document") + else: + log.log( + "fixup_bofreq_timestamps: WARNING: bofreq Document rev " + f"changed for {bofreq.name}" + ) + log.log(f"fixup_bofreq_timestamps: {bofreq.name}: " + ", ".join(log_msg_parts)) + + # Fix up the Document, if necessary, and add a record of the adjustment + DocEvent.objects.create( + type="added_comment", + by=system_person, + doc=bofreq, + rev=bofreq.rev, + desc=( + "Corrected inaccurate document and new revision event timestamps for " + + ("version " if len(adjusted_revs) == 1 else "versions ") + + ", ".join(adjusted_revs) + ), + ) From 4945809b7804c1f15de1b0340be269dd7e200f95 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 12 Feb 2026 13:56:46 -0400 Subject: [PATCH 068/158] chore(dev): update beat in docker-compose.yml (#10330) Fixes commented-out config --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2440faf121..ebe53cf95a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -129,7 +129,7 @@ services: # but can be enabled by uncommenting the following. # # beat: -# image: ghcr.io/ietf-tools/datatracker-celery:latest +# image: "${COMPOSE_PROJECT_NAME}-celery" # init: true # environment: # CELERY_APP: ietf From 8005a8baa6ffb72c47d6e35f44c0e5d78b456a2d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 13 Feb 2026 11:36:23 -0400 Subject: [PATCH 069/158] chore(dev): update docker-compose depends_on (#10410) * chore(dev): update docker-compose depends_on * chore(dev): another depends_on tweak app/celery don't actually use the blobstore container, but the Django config refers to it so we should probably depend on it anyway --- docker-compose.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ebe53cf95a..4c3f2f6b8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,9 +13,10 @@ services: # network_mode: service:db depends_on: + - blobdb + - blobstore - db - mq - - blobstore ipc: host @@ -79,7 +80,10 @@ services: command: - '--loglevel=INFO' depends_on: + - blobdb + - blobstore - db + - mq restart: unless-stopped stop_grace_period: 1m volumes: @@ -102,7 +106,10 @@ services: - '--concurrency=1' depends_on: + - blobdb + - blobstore - db + - mq restart: unless-stopped stop_grace_period: 1m volumes: From 8d804f3427b4d4c40aa6bfadba92a433bd468b26 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 20 Feb 2026 15:22:24 -0400 Subject: [PATCH 070/158] feat: button to push slide decks to Meetecho (#10431) * refactor: eliminate inline script Partially removes jQuery from this corner * chore: indicate whether slide updates were sent * feat: admin button to push slide decks to Meetecho * test: new test * test: cover interim case --- ietf/meeting/tests_views.py | 63 +++++++++++++++- ietf/meeting/urls.py | 3 +- ietf/meeting/views.py | 46 ++++++++++++ ietf/static/js/session_details.js | 53 ++++++++++++++ ietf/templates/meeting/session_details.html | 81 ++++++--------------- ietf/utils/meetecho.py | 48 +++++++++--- ietf/utils/tests_meetecho.py | 47 ++++++++++-- package.json | 1 + 8 files changed, 268 insertions(+), 74 deletions(-) create mode 100644 ietf/static/js/session_details.js diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index b94229d969..168999d0aa 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -4754,7 +4754,7 @@ def _approval_url(slidesub): 0, "second session proposed slides should be linked for approval", ) - + class EditScheduleListTests(TestCase): def setUp(self): @@ -7345,6 +7345,67 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls): fd.close() self.assertIn('third version', contents) + @override_settings( + MEETECHO_API_CONFIG="fake settings" + ) # enough to trigger API calls + @patch("ietf.meeting.views.SlidesManager") + def test_notify_meetecho_of_all_slides(self, mock_slides_manager_cls): + for meeting_type in ["ietf", "interim"]: + # Reset for the sake of the second iteration + self.client.logout() + mock_slides_manager_cls.reset_mock() + + session = SessionFactory(meeting__type_id=meeting_type) + meeting = session.meeting + + # bad meeting + url = urlreverse( + "ietf.meeting.views.notify_meetecho_of_all_slides", + kwargs={"num": 9999, "acronym": session.group.acronym}, + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + r = self.client.post(url) + self.assertEqual(r.status_code, 404) + self.assertFalse(mock_slides_manager_cls.called) + self.client.logout() + + # good meeting + url = urlreverse( + "ietf.meeting.views.notify_meetecho_of_all_slides", + kwargs={"num": meeting.number, "acronym": session.group.acronym}, + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 405) + self.assertFalse(mock_slides_manager_cls.called) + mock_slides_manager = mock_slides_manager_cls.return_value + mock_slides_manager.send_update.return_value = True + r = self.client.post(url) + self.assertEqual(r.status_code, 302) + self.assertEqual(mock_slides_manager.send_update.call_count, 1) + self.assertEqual(mock_slides_manager.send_update.call_args, call(session)) + r = self.client.get(r["Location"]) + messages = list(r.context["messages"]) + self.assertEqual(len(messages), 1) + self.assertEqual( + str(messages[0]), f"Notified Meetecho about slides for {session}" + ) + + mock_slides_manager.send_update.reset_mock() + mock_slides_manager.send_update.return_value = False + r = self.client.post(url) + self.assertEqual(r.status_code, 302) + self.assertEqual(mock_slides_manager.send_update.call_count, 1) + self.assertEqual(mock_slides_manager.send_update.call_args, call(session)) + r = self.client.get(r["Location"]) + messages = list(r.context["messages"]) + self.assertEqual(len(messages), 1) + self.assertIn( + "No sessions were eligible for Meetecho slides update.", str(messages[0]) + ) + @override_settings(IETF_NOTES_URL='https://notes.ietf.org/') class ImportNotesTests(TestCase): diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py index af36a6656c..a038e1cfe6 100644 --- a/ietf/meeting/urls.py +++ b/ietf/meeting/urls.py @@ -15,6 +15,7 @@ def get_redirect_url(self, *args, **kwargs): safe_for_all_meeting_types = [ url(r'^session/(?P[-a-z0-9]+)/?$', views.session_details), + url(r'^session/(?P[-a-z0-9]+)/send_slide_notifications$', views.notify_meetecho_of_all_slides), url(r'^session/(?P\d+)/drafts$', views.add_session_drafts), url(r'^session/(?P\d+)/recordings$', views.add_session_recordings), url(r'^session/(?P\d+)/attendance$', views.session_attendance), @@ -30,7 +31,7 @@ def get_redirect_url(self, *args, **kwargs): url(r'^session/(?P\d+)/doc/%(name)s/remove$' % settings.URL_REGEXPS, views.remove_sessionpresentation), url(r'^session/(?P\d+)\.ics$', views.agenda_ical), url(r'^sessions/(?P[-a-z0-9]+)\.ics$', views.agenda_ical), - url(r'^slidesubmission/(?P\d+)$', views.approve_proposed_slides) + url(r'^slidesubmission/(?P\d+)$', views.approve_proposed_slides), ] diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 8dccda9c87..731dfad88f 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -5710,6 +5710,52 @@ def approve_proposed_slides(request, slidesubmission_id, num): }) +@role_required("Secretariat") +def notify_meetecho_of_all_slides(request, num, acronym): + """Notify meetecho of state of all slides for the group + + Respects the usual notification window around each session. Meetecho will ignore + notices outside that window anyway, so no sense sending them. + """ + meeting = get_meeting(num=num, type_in=None) # raises 404 + if request.method != "POST": + return HttpResponseNotAllowed( + content="Method not allowed", + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + permitted_methods=("POST",), + ) + scheduled_sessions = [ + session + for session in get_sessions(meeting.number, acronym) + if session.current_status == "sched" + ] + sm = SlidesManager(api_config=settings.MEETECHO_API_CONFIG) + updated = [] + for session in scheduled_sessions: + if sm.send_update(session): + updated.append(session) + if len(updated) > 0: + messages.success( + request, + f"Notified Meetecho about slides for {','.join(str(s) for s in updated)}", + ) + elif sm.slides_notify_time is not None: + messages.warning( + request, + "No sessions were eligible for Meetecho slides update. Updates are " + f"only sent within {sm.slides_notify_time} before or after the session.", + ) + else: + messages.warning( + request, + "No sessions were eligible for Meetecho slides update. Updates are " + "currently disabled.", + ) + return redirect( + "ietf.meeting.views.session_details", num=meeting.number, acronym=acronym + ) + + def import_session_minutes(request, session_id, num): """Import session minutes from the ietf.notes.org site diff --git a/ietf/static/js/session_details.js b/ietf/static/js/session_details.js new file mode 100644 index 0000000000..03d1b2d3d9 --- /dev/null +++ b/ietf/static/js/session_details.js @@ -0,0 +1,53 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +// Relies on other scripts being loaded, see usage in session_details.html +document.addEventListener('DOMContentLoaded', () => { + // Init with best guess at local timezone. + ietf_timezone.set_tz_change_callback(timezone_changed) // cb is in upcoming.js + ietf_timezone.initialize('local') + + // Set up sortable elements if the user can manage materials + if (document.getElementById('can-manage-materials-flag')) { + const sortables = [] + const options = { + group: 'slides', + animation: 150, + handle: '.drag-handle', + onAdd: function (event) {onAdd(event)}, + onRemove: function (event) {onRemove(event)}, + onEnd: function (event) {onEnd(event)} + } + + function onAdd (event) { + const old_session = event.from.getAttribute('data-session') + const new_session = event.to.getAttribute('data-session') + $.post(event.to.getAttribute('data-add-to-session'), { + 'order': event.newIndex + 1, + 'name': event.item.getAttribute('name') + }) + $(event.item).find('td:eq(1)').find('a').each(function () { + $(this).attr('href', $(this).attr('href').replace(old_session, new_session)) + }) + } + + function onRemove (event) { + const old_session = event.from.getAttribute('data-session') + $.post(event.from.getAttribute('data-remove-from-session'), { + 'oldIndex': event.oldIndex + 1, + 'name': event.item.getAttribute('name') + }) + } + + function onEnd (event) { + if (event.to == event.from) { + $.post(event.from.getAttribute('data-reorder-in-session'), { + 'oldIndex': event.oldIndex + 1, + 'newIndex': event.newIndex + 1 + }) + } + } + + for (const elt of document.querySelectorAll('.slides tbody')) { + sortables.push(Sortable.create(elt, options)) + } + } +}) diff --git a/ietf/templates/meeting/session_details.html b/ietf/templates/meeting/session_details.html index 55fa3d3857..a4d9ba1090 100644 --- a/ietf/templates/meeting/session_details.html +++ b/ietf/templates/meeting/session_details.html @@ -1,5 +1,5 @@ {% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin ietf_filters static %} {% block title %}{{ meeting }} : {{ group.acronym }}{% endblock %} {% block morecss %} @@ -53,69 +53,36 @@

        Unscheduled Sessions

        {% endif %} {% if forloop.last %}{% endif %} {% endfor %} + {% if user|has_role:"Secretariat" %} +
        +
        + Secretariat Only +
        +
        +
        + {% csrf_token %} + + +
        +
        + {% endif %} + {% comment %} + The existence of an element with id canManageMaterialsFlag is checked in + session_details.js to determine whether it should init the sortable tables. + Not the most elegant approach, but it works. + {% endcomment %} + {% if can_manage_materials %}
        {% endif %} {% endblock %} {% block js %} - {% if can_manage_materials %} - {% endif %} + {% endblock %} \ No newline at end of file diff --git a/ietf/utils/meetecho.py b/ietf/utils/meetecho.py index 7654f67cd1..943f3789ef 100644 --- a/ietf/utils/meetecho.py +++ b/ietf/utils/meetecho.py @@ -508,8 +508,13 @@ def _should_send_update(self, session): return (timeslot.time - self.slides_notify_time) < now < (timeslot.end_time() + self.slides_notify_time) def add(self, session: "Session", slides: "Document", order: int): + """Add a slide deck to the session + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ if not self._should_send_update(session): - return + return False # Would like to confirm that session.presentations includes the slides Document, but we can't # (same problem regarding unsaved Documents discussed in the docstring) @@ -524,11 +529,16 @@ def add(self, session: "Session", slides: "Document", order: int): "order": order, } ) + return True def delete(self, session: "Session", slides: "Document"): - """Delete a slide deck from the session""" + """Delete a slide deck from the session + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ if not self._should_send_update(session): - return + return False if session.presentations.filter(document=slides).exists(): # "order" problems are very likely to result if we delete slides that are actually still @@ -543,12 +553,17 @@ def delete(self, session: "Session", slides: "Document"): id=slides.pk, ) if session.presentations.filter(document__type_id="slides").exists(): - self.send_update(session) # adjust order to fill in the hole + self._send_update(session) # adjust order to fill in the hole + return True def revise(self, session: "Session", slides: "Document"): - """Replace existing deck with its current state""" + """Replace existing deck with its current state + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ if not self._should_send_update(session): - return + return False sp = session.presentations.filter(document=slides).first() if sp is None: @@ -561,11 +576,13 @@ def revise(self, session: "Session", slides: "Document"): id=slides.pk, ) self.add(session, slides, order) # fill in the hole + return True - def send_update(self, session: "Session"): - if not self._should_send_update(session): - return - + def _send_update(self, session: "Session"): + """Notify of the current state of the session's slides (no time window check) + + This is a private helper - use send_update() (no leading underscore) instead. + """ self.api.update_slide_decks( wg_token=self.wg_token(session.group), session=str(session.pk), @@ -580,3 +597,14 @@ def send_update(self, session: "Session"): for deck in session.presentations.filter(document__type="slides") ] ) + + def send_update(self, session: "Session"): + """Notify of the current state of the session's slides + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ + if not self._should_send_update(session): + return False + self._send_update(session) + return True diff --git a/ietf/utils/tests_meetecho.py b/ietf/utils/tests_meetecho.py index 502e936483..c076a3df74 100644 --- a/ietf/utils/tests_meetecho.py +++ b/ietf/utils/tests_meetecho.py @@ -547,7 +547,8 @@ def test_add(self, mock_add, mock_wg_token): sm = SlidesManager(settings.MEETECHO_API_CONFIG) session = SessionFactory() slides_doc = DocumentFactory(type_id="slides") - sm.add(session, slides_doc, 13) + retval = sm.add(session, slides_doc, 13) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_add.called) self.assertEqual( @@ -565,6 +566,14 @@ def test_add(self, mock_add, mock_wg_token): ), ) + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.add(session, slides_doc, 14) + self.assertIs(retval, False) + @patch("ietf.utils.meetecho.MeetechoAPI.update_slide_decks") @patch("ietf.utils.meetecho.MeetechoAPI.delete_slide_deck") def test_delete(self, mock_delete, mock_update, mock_wg_token): @@ -580,7 +589,8 @@ def test_delete(self, mock_delete, mock_update, mock_wg_token): sm.delete(session, slides_doc) # can't remove slides still attached to the session self.assertFalse(any([mock_wg_token.called, mock_delete.called, mock_update.called])) - sm.delete(session, removed_slides_doc) + retval = sm.delete(session, removed_slides_doc) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_delete.called) self.assertEqual( @@ -609,9 +619,18 @@ def test_delete(self, mock_delete, mock_update, mock_wg_token): # Delete the other session and check that we don't make the update call slides.delete() - sm.delete(session, slides_doc) + retval = sm.delete(session, slides_doc) + self.assertIs(retval, True) self.assertTrue(mock_delete.called) self.assertFalse(mock_update.called) + + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.delete(session, slides_doc) + self.assertIs(retval, False) @patch("ietf.utils.meetecho.MeetechoAPI.delete_slide_deck") @patch("ietf.utils.meetecho.MeetechoAPI.add_slide_deck") @@ -619,7 +638,8 @@ def test_revise(self, mock_add, mock_delete, mock_wg_token): sm = SlidesManager(settings.MEETECHO_API_CONFIG) slides = SessionPresentationFactory(document__type_id="slides", order=23) slides_doc = slides.document - sm.revise(slides.session, slides.document) + retval = sm.revise(slides.session, slides_doc) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_delete.called) self.assertEqual( @@ -642,13 +662,22 @@ def test_revise(self, mock_add, mock_delete, mock_wg_token): ), ) + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.revise(slides.session, slides_doc) + self.assertIs(retval, False) + @patch("ietf.utils.meetecho.MeetechoAPI.update_slide_decks") def test_send_update(self, mock_send_update, mock_wg_token): sm = SlidesManager(settings.MEETECHO_API_CONFIG) slides = SessionPresentationFactory(document__type_id="slides") SessionPresentationFactory(session=slides.session, document__type_id="agenda") - sm.send_update(slides.session) + retval = sm.send_update(slides.session) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_send_update.called) self.assertEqual( @@ -667,3 +696,11 @@ def test_send_update(self, mock_send_update, mock_wg_token): ] ) ) + + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.send_update(slides.session) + self.assertIs(retval, False) diff --git a/package.json b/package.json index e2e6fd7dab..fec29275b4 100644 --- a/package.json +++ b/package.json @@ -148,6 +148,7 @@ "ietf/static/js/moment.js", "ietf/static/js/password_strength.js", "ietf/static/js/select2.js", + "ietf/static/js/session_details.js", "ietf/static/js/session_details_form.js", "ietf/static/js/session_form.js", "ietf/static/js/session_request.js", From 619b2aee0f3b4acbc95a44f6fd3e8785163f6a93 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 20 Feb 2026 13:22:49 -0600 Subject: [PATCH 071/158] fix: adjust draft-stream-ietf state descriptions per IESG (#10437) --- ...ge_draft_stream_ietf_state_descriptions.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py diff --git a/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py b/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py new file mode 100644 index 0000000000..c664126da3 --- /dev/null +++ b/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py @@ -0,0 +1,57 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations + + +def forward(apps, schema_editor): + State = apps.get_model("doc", "State") + for name, desc in [ + ( + "Adopted by a WG", + "The individual submission document has been adopted by the Working Group (WG), but some administrative matter still needs to be completed (e.g., a WG document replacing this document with the typical naming convention of 'draft-ietf-wgname-topic-nn' has not yet been submitted).", + ), + ( + "WG Document", + "The document has been identified as a Working Group (WG) document and is under development per Section 7.2 of RFC2418.", + ), + ( + "Waiting for WG Chair Go-Ahead", + "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chairs are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed.", + ), + ( + "Submitted to IESG for Publication", + "The Working Group (WG) document has been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication per Section 7.4 of RFC2418. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", + ), + ]: + State.objects.filter(name=name).update(desc=desc, type="draft-stream-ietf") + + +def reverse(apps, schema_editor): + State = apps.get_model("doc", "State") + for name, desc in [ + ( + "Adopted by a WG", + "The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted.", + ), + ( + "WG Document", + "The document has been adopted by the Working Group (WG) and is under development. A document can only be adopted by one WG at a time. However, a document may be transferred between WGs.", + ), + ( + "Waiting for WG Chair Go-Ahead", + "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed", + ), + ( + "Submitted to IESG for Publication", + "The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", + ), + ]: + State.objects.filter(name=name).update(desc=desc, type="draft-stream-ietf") + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0030_alter_dochistory_title_alter_document_title"), + ] + + operations = [migrations.RunPython(forward, reverse)] From c4be6318f73cbf896b5cc1f3416040e12b4611f4 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 20 Feb 2026 15:25:57 -0400 Subject: [PATCH 072/158] feat: more API fields+filtering; drop RfcAuthor.email field (#10432) * feat: RfcAuthorSerializer.email is current email * refactor: RfcAuthor email field -> property * feat: more RfcMetadataSerializer fields * shepherd email (with a fallback to the draft) * doc ad email * area ad emails * group list email * fix: filter RFCs by any group type * feat: filter by RFC numbers * fix: shepherd -> draft object in response JSON * fix: consistent filter naming * chore: migration * test: update test_notify_rfc_published * fix: RfcAuthor.email() -> Email, not str * fix: update RfcAuthorFactory * fix: consistent blank value in email() * fix: guard against non-prefetched queryset * test: fix nomcom test * refactor: name-addr -> addr for ad/shepherd Also falls back to current primary email for ad/shepherd if the email on record is inactive. --- ietf/api/tests_views_rpc.py | 25 +++++----- ietf/doc/admin.py | 4 +- ietf/doc/api.py | 10 +++- ietf/doc/factories.py | 1 - .../migrations/0031_remove_rfcauthor_email.py | 16 ++++++ ietf/doc/models.py | 6 ++- ietf/doc/serializers.py | 50 +++++++++++++++---- ietf/doc/views_doc.py | 2 +- ietf/group/serializers.py | 32 ++++++++++-- ietf/nomcom/tests.py | 1 - 10 files changed, 115 insertions(+), 32 deletions(-) create mode 100644 ietf/doc/migrations/0031_remove_rfcauthor_email.py diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 09fb40bf6e..1fbb4c3f02 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -143,22 +143,23 @@ def test_notify_rfc_published(self): self.assertEqual(rfc.title, "RFC " + draft.title) self.assertEqual(rfc.documentauthor_set.count(), 0) self.assertEqual( - list( - rfc.rfcauthor_set.values( - "titlepage_name", - "is_editor", - "person", - "email", - "affiliation", - "country", - ) - ), + [ + { + "titlepage_name": ra.titlepage_name, + "is_editor": ra.is_editor, + "person": ra.person, + "email": ra.email, + "affiliation": ra.affiliation, + "country": ra.country, + } + for ra in rfc.rfcauthor_set.all() + ], [ { "titlepage_name": f"titlepage {author.name}", "is_editor": False, - "person": author.pk, - "email": author.email_address(), + "person": author, + "email": author.email(), "affiliation": "Some Affiliation", "country": "CA", } diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index f082418935..b604d4f096 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -242,6 +242,6 @@ def is_deleted(self, instance): class RfcAuthorAdmin(admin.ModelAdmin): list_display = ['id', 'document', 'titlepage_name', 'person', 'email', 'affiliation', 'country', 'order'] - search_fields = ['document__name', 'titlepage_name', 'person__name', 'email__address', 'affiliation', 'country'] - raw_id_fields = ["document", "person", "email"] + search_fields = ['document__name', 'titlepage_name', 'person__name', 'email', 'affiliation', 'country'] + raw_id_fields = ["document", "person"] admin.site.register(RfcAuthor, RfcAuthorAdmin) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 6a4c0c9fd5..75993f463e 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -42,13 +42,21 @@ class RfcLimitOffsetPagination(LimitOffsetPagination): max_limit = 500 +class NumberInFilter(filters.BaseInFilter, filters.NumberFilter): + """Filter against a comma-separated list of numbers""" + pass + + class RfcFilter(filters.FilterSet): published = filters.DateFromToRangeFilter() stream = filters.ModelMultipleChoiceFilter( queryset=StreamName.objects.filter(used=True) ) + number = NumberInFilter( + field_name="rfc_number" + ) group = filters.ModelMultipleChoiceFilter( - queryset=Group.objects.wgs(), + queryset=Group.objects.all(), field_name="group__acronym", to_field_name="acronym", ) diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index aad01be04f..bc38765446 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -391,7 +391,6 @@ class Meta: lambda obj: " ".join([obj.person.initials(), obj.person.last_name()]) ) person = factory.SubFactory('ietf.person.factories.PersonFactory') - email = factory.LazyAttribute(lambda obj: obj.person.email()) affiliation = factory.Faker('company') order = factory.LazyAttribute(lambda o: o.document.rfcauthor_set.count() + 1) diff --git a/ietf/doc/migrations/0031_remove_rfcauthor_email.py b/ietf/doc/migrations/0031_remove_rfcauthor_email.py new file mode 100644 index 0000000000..c4c1911bfe --- /dev/null +++ b/ietf/doc/migrations/0031_remove_rfcauthor_email.py @@ -0,0 +1,16 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0030_alter_dochistory_title_alter_document_title"), + ] + + operations = [ + migrations.RemoveField( + model_name="rfcauthor", + name="email", + ), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 8f700bf496..cc28951be0 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -937,7 +937,6 @@ class RfcAuthor(models.Model): titlepage_name = models.CharField(max_length=128, blank=False) is_editor = models.BooleanField(default=False) person = ForeignKey(Person, null=True, blank=True, on_delete=models.PROTECT) - email = ForeignKey(Email, help_text="Email address used by author for submission", blank=True, null=True, on_delete=models.PROTECT) affiliation = models.CharField(max_length=100, blank=True, help_text="Organization/company used by author for submission") country = models.CharField(max_length=255, blank=True, help_text="Country used by author for submission") order = models.IntegerField(default=1) @@ -951,6 +950,11 @@ class Meta: models.Index(fields=["document", "order"]) ] + @property + def email(self) -> Email | None: + return self.person.email() if self.person else None + + class DocumentAuthorInfo(models.Model): person = ForeignKey(Person) # email should only be null for some historic documents diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index e8d373164b..b054b074d7 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -9,14 +9,20 @@ from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from ietf.group.serializers import GroupSerializer +from ietf.group.serializers import ( + AreaDirectorSerializer, + AreaSerializer, + GroupSerializer, +) from ietf.name.serializers import StreamNameSerializer +from ietf.utils import log from .models import Document, DocumentAuthor, RfcAuthor class RfcAuthorSerializer(serializers.ModelSerializer): """Serializer for an RfcAuthor / DocumentAuthor in a response""" + email = serializers.EmailField(source="email.address", read_only=True) datatracker_person_path = serializers.URLField( source="person.get_absolute_url", required=False, @@ -29,7 +35,7 @@ class Meta: "titlepage_name", "is_editor", "person", - "email", # relies on email.pk being email.address + "email", "affiliation", "country", "datatracker_person_path", @@ -48,7 +54,6 @@ def to_representation(self, instance): titlepage_name=document_author.person.plain_name(), is_editor=False, person=document_author.person, - email=document_author.email, affiliation=document_author.affiliation, country=document_author.country, order=document_author.order, @@ -174,10 +179,16 @@ def to_representation(self, instance: Document): return super().to_representation(instance=RfcStatus.from_document(instance)) +class ShepherdSerializer(serializers.Serializer): + email = serializers.EmailField(source="email_address") + + class RelatedDraftSerializer(serializers.Serializer): id = serializers.IntegerField(source="source.id") name = serializers.CharField(source="source.name") title = serializers.CharField(source="source.title") + shepherd = ShepherdSerializer(source="source.shepherd") + ad = AreaDirectorSerializer(source="source.ad") class RelatedRfcSerializer(serializers.Serializer): @@ -205,15 +216,23 @@ class RfcFormatSerializer(serializers.Serializer): class RfcMetadataSerializer(serializers.ModelSerializer): - """Serialize metadata of an RFC""" + """Serialize metadata of an RFC + + This needs to be called with a Document queryset that has been processed with + api.augment_rfc_queryset() or it very likely will not work. Some of the typing + refers to Document, but this should really be WithAnnotations[Document, ...]. + However, have not been able to make that work yet. + """ number = serializers.IntegerField(source="rfc_number") published = serializers.DateField() status = RfcStatusSerializer(source="*") authors = serializers.SerializerMethodField() group = GroupSerializer() - area = GroupSerializer(source="group.area", required=False) + area = AreaSerializer(source="group.area", required=False) stream = StreamNameSerializer() + ad = AreaDirectorSerializer(read_only=True) + group_list_email = serializers.EmailField(source="group.list_email", read_only=True) identifiers = serializers.SerializerMethodField() draft = serializers.SerializerMethodField() obsoletes = RelatedRfcSerializer(many=True, read_only=True) @@ -239,6 +258,8 @@ class Meta: "group", "area", "stream", + "ad", + "group_list_email", "identifiers", "obsoletes", "obsoleted_by", @@ -276,11 +297,20 @@ def get_identifiers(self, doc: Document): return DocIdentifierSerializer(instance=identifiers, many=True).data @extend_schema_field(RelatedDraftSerializer) - def get_draft(self, object): - try: - related_doc = object.drafts[0] - except IndexError: - return None + def get_draft(self, doc: Document): + if hasattr(doc, "drafts"): + # This is the expected case - drafts is added by a Prefetch in + # the augment_rfc_queryset() method. + try: + related_doc = doc.drafts[0] + except IndexError: + return None + else: + # Fallback in case augment_rfc_queryset() was not called + log.log( + f"Warning: {self.__class__}.get_draft() called without prefetched draft" + ) + related_doc = doc.came_from_draft() return RelatedDraftSerializer(related_doc).data diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 0578da1b77..0ae7520681 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -1657,7 +1657,7 @@ def extract_name(s): doc.rfcauthor_set if doc.type_id == "rfc" and doc.rfcauthor_set.exists() else doc.documentauthor_set - ).select_related("person", "email").order_by("order") + ).select_related("person").prefetch_related("person__email_set").order_by("order") data["authors"] = [ { "name": author.titlepage_name if hasattr(author, "titlepage_name") else author.person.name, diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py index 08e6bba81a..85f209019c 100644 --- a/ietf/group/serializers.py +++ b/ietf/group/serializers.py @@ -1,11 +1,37 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved """django-rest-framework serializers""" + +from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from .models import Group +from ietf.person.models import Email +from .models import Group, Role class GroupSerializer(serializers.ModelSerializer): class Meta: model = Group - fields = ["acronym", "name", "type"] + fields = ["acronym", "name", "type", "list_email"] + + +class AreaDirectorSerializer(serializers.Serializer): + """Serialize an area director + + Works with Email or Role + """ + + email = serializers.SerializerMethodField() + + @extend_schema_field(serializers.EmailField) + def get_email(self, instance: Email | Role): + if isinstance(instance, Role): + return instance.email.email_address() + return instance.email_address() + + +class AreaSerializer(serializers.ModelSerializer): + ads = AreaDirectorSerializer(many=True, read_only=True) + + class Meta: + model = Group + fields = ["acronym", "name", "type", "ads"] diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index b6e8c57da7..210788ce07 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -2528,7 +2528,6 @@ def test_get_qualified_author_queryset(self): document=rfc, person=people[0], titlepage_name="P. Zero", - email=people[0].email_set.first(), ) self.assertCountEqual( get_qualified_author_queryset(base_qs, now - 5 * one_year, now), From d7319030f3d3a38bf0c048e713bc1f068ca228ed Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 20 Feb 2026 16:12:43 -0400 Subject: [PATCH 073/158] chore: renumber migrations (#10441) --- ...remove_rfcauthor_email.py => 0032_remove_rfcauthor_email.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename ietf/doc/migrations/{0031_remove_rfcauthor_email.py => 0032_remove_rfcauthor_email.py} (80%) diff --git a/ietf/doc/migrations/0031_remove_rfcauthor_email.py b/ietf/doc/migrations/0032_remove_rfcauthor_email.py similarity index 80% rename from ietf/doc/migrations/0031_remove_rfcauthor_email.py rename to ietf/doc/migrations/0032_remove_rfcauthor_email.py index c4c1911bfe..a0e147da59 100644 --- a/ietf/doc/migrations/0031_remove_rfcauthor_email.py +++ b/ietf/doc/migrations/0032_remove_rfcauthor_email.py @@ -5,7 +5,7 @@ class Migration(migrations.Migration): dependencies = [ - ("doc", "0030_alter_dochistory_title_alter_document_title"), + ("doc", "0031_change_draft_stream_ietf_state_descriptions"), ] operations = [ From abf5e0d97ca38ede129b218d3a895f9bb5ab441b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 20 Feb 2026 17:45:31 -0400 Subject: [PATCH 074/158] fix: allow null for shepherd/ad (#10443) --- ietf/doc/serializers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index b054b074d7..e42a6a0293 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -187,8 +187,8 @@ class RelatedDraftSerializer(serializers.Serializer): id = serializers.IntegerField(source="source.id") name = serializers.CharField(source="source.name") title = serializers.CharField(source="source.title") - shepherd = ShepherdSerializer(source="source.shepherd") - ad = AreaDirectorSerializer(source="source.ad") + shepherd = ShepherdSerializer(source="source.shepherd", allow_null=True) + ad = AreaDirectorSerializer(source="source.ad", allow_null=True) class RelatedRfcSerializer(serializers.Serializer): @@ -231,7 +231,7 @@ class RfcMetadataSerializer(serializers.ModelSerializer): group = GroupSerializer() area = AreaSerializer(source="group.area", required=False) stream = StreamNameSerializer() - ad = AreaDirectorSerializer(read_only=True) + ad = AreaDirectorSerializer(read_only=True, allow_null=True) group_list_email = serializers.EmailField(source="group.list_email", read_only=True) identifiers = serializers.SerializerMethodField() draft = serializers.SerializerMethodField() From 18902ff1be5746cafc958561e563f00be5f05176 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 24 Feb 2026 13:10:42 -0400 Subject: [PATCH 075/158] fix: more accurate group areas (red API) (#10462) * fix: RFC area only for ietf stream * fix: no ADs for inactive areas --- ietf/doc/serializers.py | 19 ++++++++++++++++++- ietf/group/serializers.py | 9 ++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index e42a6a0293..36076c30be 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -229,7 +229,7 @@ class RfcMetadataSerializer(serializers.ModelSerializer): status = RfcStatusSerializer(source="*") authors = serializers.SerializerMethodField() group = GroupSerializer() - area = AreaSerializer(source="group.area", required=False) + area = serializers.SerializerMethodField() stream = StreamNameSerializer() ad = AreaDirectorSerializer(read_only=True, allow_null=True) group_list_email = serializers.EmailField(source="group.list_email", read_only=True) @@ -287,6 +287,23 @@ def get_authors(self, doc: Document): many=True, ).data + @extend_schema_field(AreaSerializer(required=False)) + def get_area(self, doc: Document): + """Get area for the RFC + + This logic might be better moved to Document or a combination of Document + and Group. The current (2026-02-24) Group.area() method is not strict enough: + it does not limit to WG groups or IETF-stream documents. + """ + if doc.stream_id != "ietf": + return None + if doc.group is None: + return None + parent = doc.group.parent + if parent.type_id == "area": + return AreaSerializer(parent).data + return None + @extend_schema_field(DocIdentifierSerializer(many=True)) def get_identifiers(self, doc: Document): identifiers = [] diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py index 85f209019c..4ba92232c5 100644 --- a/ietf/group/serializers.py +++ b/ietf/group/serializers.py @@ -30,8 +30,15 @@ def get_email(self, instance: Email | Role): class AreaSerializer(serializers.ModelSerializer): - ads = AreaDirectorSerializer(many=True, read_only=True) + ads = serializers.SerializerMethodField() class Meta: model = Group fields = ["acronym", "name", "type", "ads"] + + @extend_schema_field(AreaDirectorSerializer(many=True)) + def get_ads(self, area: Group): + return AreaDirectorSerializer( + area.ads() if area.is_active else Role.objects.none(), + many=True, + ).data From 07efd2b078a461da2eb7e197fc91f2ae0b45ac40 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 25 Feb 2026 12:58:57 -0400 Subject: [PATCH 076/158] fix: unbreak red API + group serializer tests (#10467) * test: group serializer tests * fix: Group.ads is a property * fix: no need for type in AreaSerializer --- ietf/group/serializers.py | 4 +- ietf/group/tests_serializers.py | 90 +++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 ietf/group/tests_serializers.py diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py index 4ba92232c5..db3b37af48 100644 --- a/ietf/group/serializers.py +++ b/ietf/group/serializers.py @@ -34,11 +34,11 @@ class AreaSerializer(serializers.ModelSerializer): class Meta: model = Group - fields = ["acronym", "name", "type", "ads"] + fields = ["acronym", "name", "ads"] @extend_schema_field(AreaDirectorSerializer(many=True)) def get_ads(self, area: Group): return AreaDirectorSerializer( - area.ads() if area.is_active else Role.objects.none(), + area.ads if area.is_active else Role.objects.none(), many=True, ).data diff --git a/ietf/group/tests_serializers.py b/ietf/group/tests_serializers.py new file mode 100644 index 0000000000..bf29e6c8fd --- /dev/null +++ b/ietf/group/tests_serializers.py @@ -0,0 +1,90 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from ietf.group.factories import RoleFactory, GroupFactory +from ietf.group.serializers import ( + AreaDirectorSerializer, + AreaSerializer, + GroupSerializer, +) +from ietf.person.factories import EmailFactory +from ietf.utils.test_utils import TestCase + + +class GroupSerializerTests(TestCase): + def test_serializes(self): + wg = GroupFactory() + serialized = GroupSerializer(wg).data + self.assertEqual( + serialized, + { + "acronym": wg.acronym, + "name": wg.name, + "type": "wg", + "list_email": wg.list_email, + }, + ) + + +class AreaDirectorSerializerTests(TestCase): + def test_serializes_role(self): + """Should serialize a Role correctly""" + role = RoleFactory(group__type_id="area", name_id="ad") + serialized = AreaDirectorSerializer(role).data + self.assertEqual( + serialized, + {"email": role.email.email_address()}, + ) + + def test_serializes_email(self): + """Should serialize an Email correctly""" + email = EmailFactory() + serialized = AreaDirectorSerializer(email).data + self.assertEqual( + serialized, + {"email": email.email_address()}, + ) + + +class AreaSerializerTests(TestCase): + def test_serializes_active_area(self): + """Should serialize an active area correctly""" + area = GroupFactory(type_id="area", state_id="active") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) + ad_roles = RoleFactory.create_batch(2, group=area, name_id="ad") + serialized = AreaSerializer(area).data + self.assertEqual(serialized["acronym"], area.acronym) + self.assertEqual(serialized["name"], area.name) + self.assertCountEqual( + serialized["ads"], + [{"email": ad.email.email_address()} for ad in ad_roles], + ) + + def test_serializes_inactive_area(self): + """Should serialize an inactive area correctly""" + area = GroupFactory(type_id="area", state_id="conclude") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) + RoleFactory.create_batch(2, group=area, name_id="ad") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) From b81249884877e20c6e311478fe25b472c869c555 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 25 Feb 2026 13:24:30 -0600 Subject: [PATCH 077/158] feat: notify errata system of new rfc (#10465) * feat: notify errata system of new rfc * chore: ruff --- docker/configs/settings_local.py | 4 ++++ ietf/api/tests_views_rpc.py | 28 +++++++++++++++++++------ ietf/api/views_rpc.py | 9 +++++++- ietf/doc/tasks.py | 5 +++++ ietf/doc/utils_errata.py | 35 ++++++++++++++++++++++++++++++++ ietf/settings.py | 5 +++++ 6 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 ietf/doc/utils_errata.py diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index e357ce3f73..1d4e6916b9 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -105,3 +105,7 @@ "ietf.api.red_api" : ["devtoken", "redtoken"], # Not a real secret "ietf.api.views_rpc" : ["devtoken"], # Not a real secret } + +# Errata system api configuration +ERRATA_METADATA_NOTIFICATION_URL = "http://host.docker.internal:8808/api/rfc_metadata_update/" +ERRATA_METADATA_NOTIFICATION_API_KEY = "not a real secret" diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 1fbb4c3f02..6a5a5c9b88 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -9,9 +9,10 @@ from django.db.models.functions import Coalesce from django.test.utils import override_settings from django.urls import reverse as urlreverse +import mock from ietf.blobdb.models import Blob -from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory +from ietf.doc.factories import IndividualDraftFactory, RfcFactory, WgDraftFactory, WgRfcFactory from ietf.doc.models import RelatedDocument, Document from ietf.group.factories import RoleFactory, GroupFactory from ietf.person.factories import PersonFactory @@ -77,7 +78,8 @@ def test_draftviewset_references(self): self.assertEqual(refs[0]["name"], draft_bar.name) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) - def test_notify_rfc_published(self): + @mock.patch("ietf.doc.tasks.signal_update_rfc_metadata_task.delay") + def test_notify_rfc_published(self, mock_task_delay): url = urlreverse("ietf.api.purple_api.notify_rfc_published") area = GroupFactory(type_id="area") rfc_group = GroupFactory(type_id="wg") @@ -90,6 +92,8 @@ def test_notify_rfc_published(self): ) rfc_stream_id = "ise" assert isinstance(draft, Document), "WgDraftFactory should generate a Document" + updates = RfcFactory.create_batch(2) + obsoletes = RfcFactory.create_batch(2) unused_rfc_number = ( Document.objects.filter(rfc_number__isnull=False).aggregate( unused_rfc_number=Max("rfc_number") + 1 @@ -120,8 +124,8 @@ def test_notify_rfc_published(self): "pages": draft.pages + 10, "std_level": "ps", "ad": rfc_ad.pk, - "obsoletes": [], - "updates": [], + "obsoletes": [o.rfc_number for o in obsoletes], + "updates": [o.rfc_number for o in updates], "subseries": [], } r = self.client.post(url, data=post_data, format="json") @@ -172,13 +176,25 @@ def test_notify_rfc_published(self): self.assertEqual(rfc.pages, draft.pages + 10) self.assertEqual(rfc.std_level_id, "ps") self.assertEqual(rfc.ad, rfc_ad) - self.assertEqual(rfc.related_that_doc("obs"), []) - self.assertEqual(rfc.related_that_doc("updates"), []) + self.assertEqual(set(rfc.related_that_doc("obs")), set([o for o in obsoletes])) + self.assertEqual( + set(rfc.related_that_doc("updates")), set([o for o in updates]) + ) self.assertEqual(rfc.part_of(), []) self.assertEqual(draft.get_state().slug, "rfc") # todo test non-empty relationships # todo test references (when updating that is part of the handling) + self.assertTrue(mock_task_delay.called) + mock_args, mock_kwargs = mock_task_delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_rfc_number_list = [rfc.rfc_number] + expected_rfc_number_list.extend( + [d.rfc_number for d in updates + obsoletes] + ) + expected_rfc_number_list = sorted(set(expected_rfc_number_list)) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) def test_upload_rfc_files(self): def _valid_post_data(): diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 2bf16480f2..9273590b28 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -38,6 +38,7 @@ from ietf.doc.models import Document, DocHistory, RfcAuthor from ietf.doc.serializers import RfcAuthorSerializer from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage +from ietf.doc.tasks import signal_update_rfc_metadata_task from ietf.person.models import Email, Person @@ -362,7 +363,7 @@ def post(self, request): serializer.is_valid(raise_exception=True) # Create RFC try: - serializer.save() + rfc = serializer.save() except IntegrityError as err: if Document.objects.filter( rfc_number=serializer.validated_data["rfc_number"] @@ -375,6 +376,12 @@ def post(self, request): f"Unable to publish: {err}", code="unknown-integrity-error", ) + rfc_number_list = [rfc.rfc_number] + rfc_number_list.extend( + [d.rfc_number for d in rfc.related_that_doc(("updates", "obs"))] + ) + rfc_number_list = sorted(set(rfc_number_list)) + signal_update_rfc_metadata_task.delay(rfc_number_list=rfc_number_list) return Response(NotificationAckSerializer().data) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index b463b9cecf..90f4c80af5 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -35,6 +35,7 @@ investigate_fragment, ) from .utils_bofreq import fixup_bofreq_timestamps +from .utils_errata import signal_update_rfc_metadata @shared_task @@ -155,3 +156,7 @@ def rebuild_reference_relations_task(doc_names: list[str]): @shared_task def fixup_bofreq_timestamps_task(): # pragma: nocover fixup_bofreq_timestamps() + +@shared_task +def signal_update_rfc_metadata_task(rfc_number_list=()): + signal_update_rfc_metadata(rfc_number_list) diff --git a/ietf/doc/utils_errata.py b/ietf/doc/utils_errata.py new file mode 100644 index 0000000000..539262151f --- /dev/null +++ b/ietf/doc/utils_errata.py @@ -0,0 +1,35 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import requests + +from django.conf import settings + +from ietf.utils.log import log + + +def signal_update_rfc_metadata(rfc_number_list=()): + key = getattr(settings, "ERRATA_METADATA_NOTIFICATION_API_KEY", None) + if key is not None: + headers = {"X-Api-Key": settings.ERRATA_METADATA_NOTIFICATION_API_KEY} + post_dict = { + "rfc_number_list": list(rfc_number_list), + } + try: + response = requests.post( + settings.ERRATA_METADATA_NOTIFICATION_URL, + headers=headers, + json=post_dict, + timeout=settings.DEFAULT_REQUESTS_TIMEOUT, + ) + except requests.Timeout as e: + log( + f"POST request timed out for {settings.ERRATA_METADATA_NOTIFICATION_URL} ]: {e}" + ) + # raise RuntimeError(f'POST request timed out for {settings.ERRATA_METADATA_NOTIFICATION_URL}') from e + return + if response.status_code != 200: + log( + f"POST request failed for {settings.ERRATA_METADATA_NOTIFICATION_URL} ]: {response.status_code} {response.text}" + ) + else: + log("No API key configured for errata metadata notification, skipping") diff --git a/ietf/settings.py b/ietf/settings.py index 565e8825a9..71b110d762 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1368,6 +1368,11 @@ def skip_unreadable_post(record): MEETECHO_AUDIO_STREAM_URL = "https://mp3.conf.meetecho.com/ietf{session.meeting.number}/{session.pk}.m3u" MEETECHO_SESSION_RECORDING_URL = "https://meetecho-player.ietf.org/playout/?session={session_label}" +# Errata system api configuration +# settings should provide +# ERRATA_METADATA_NOTIFICATION_URL +# ERRATA_METADATA_NOTIFICATION_API_KEY + # Put the production SECRET_KEY in settings_local.py, and also any other # sensitive or site-specific changes. DO NOT commit settings_local.py to svn. from ietf.settings_local import * # pyflakes:ignore pylint: disable=wildcard-import From da5614c4963c3dc4ff8e901c1edd888296219a0d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 25 Feb 2026 18:14:47 -0400 Subject: [PATCH 078/158] test: avoid random fail in test_rfc_index (#10469) --- ietf/sync/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index bcc87a43aa..21d6cb5cd5 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -301,6 +301,7 @@ def test_rfc_index(self): ad=Person.objects.get(user__username='ad'), external_url="http://my-external-url.example.com", note="this is a note", + pages=54, # make sure this is not 42 ) DocumentAuthorFactory.create_batch(2, document=draft_doc) draft_doc.action_holders.add(draft_doc.ad) # not normally set, but add to be sure it's cleared From c1c24d012d23135725f0206dbe1a6be1e2a7fef4 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 27 Feb 2026 14:31:40 -0400 Subject: [PATCH 079/158] feat: RFC metadata update API (#10476) * feat: more editable RFC fields for API (WIP) Checkpoint commit! * chore: avoid requiring prefetch Makes some fields write-only to achieve this. * refactor: replace EditableRfcSerializer * fix: mark read-only field properly * refactor: SubseriesNameField * test: EditableRfcSerializer * refactor: DocEvent adjustment * feat: record person ids for authors * chore: adjust history message * fix: always save!! * fix: better msg formatting * fix: _almost_ always save!! * fix: lint * refactor: rename var --- ietf/api/serializers_rpc.py | 227 ++++++++++++++++++++++++++---- ietf/api/tests_serializers_rpc.py | 139 ++++++++++++++++++ ietf/api/views_rpc.py | 12 +- ietf/doc/serializers.py | 1 + ietf/doc/utils.py | 16 ++- 5 files changed, 361 insertions(+), 34 deletions(-) create mode 100644 ietf/api/tests_serializers_rpc.py diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 34e2c791c0..d5f5363990 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -216,32 +216,24 @@ class Meta: read_only_fields = ["id", "name"] -class EditableRfcSerializer(serializers.ModelSerializer): - # Would be nice to reconcile this with ietf.doc.serializers.RfcSerializer. - # The purposes of that serializer (representing data for Red) and this one - # (accepting updates from Purple) are different enough that separate formats - # may be needed, but if not it'd be nice to have a single RfcSerializer that - # can serve both. - # - # For now, only handles authors - authors = RfcAuthorSerializer(many=True, min_length=1, source="rfcauthor_set") +def _update_authors(rfc, authors_data): + # Construct unsaved instances from validated author data + new_authors = [RfcAuthor(**authdata) for authdata in authors_data] + # Update the RFC with the new author set + with transaction.atomic(): + change_events = update_rfcauthors(rfc, new_authors) + for event in change_events: + event.save() + return change_events - class Meta: - model = Document - fields = ["id", "authors"] - def update(self, instance, validated_data): - assert isinstance(instance, Document) - authors_data = validated_data.pop("rfcauthor_set", None) - if authors_data is not None: - # Construct unsaved instances from validated author data - new_authors = [RfcAuthor(**ad) for ad in authors_data] - # Update the RFC with the new author set - with transaction.atomic(): - change_events = update_rfcauthors(instance, new_authors) - for event in change_events: - event.save() - return instance +class SubseriesNameField(serializers.RegexField): + + def __init__(self, **kwargs): + # pattern: no leading 0, finite length (arbitrarily set to 5 digits) + regex = r"^(bcp|std|fyi)[1-9][0-9]{0,4}$" + super().__init__(regex, **kwargs) + class RfcPubSerializer(serializers.ModelSerializer): @@ -283,13 +275,7 @@ class RfcPubSerializer(serializers.ModelSerializer): slug_field="rfc_number", queryset=Document.objects.filter(type_id="rfc"), ) - subseries = serializers.ListField( - child=serializers.RegexField( - required=False, - # pattern: no leading 0, finite length (arbitrarily set to 5 digits) - regex=r"^(bcp|std|fyi)[1-9][0-9]{0,4}$", - ) - ) + subseries = serializers.ListField(child=SubseriesNameField(required=False)) # N.b., authors is _not_ a field on Document! authors = RfcAuthorSerializer(many=True) @@ -327,6 +313,9 @@ def validate(self, data): ) return data + def update(self, instance, validated_data): + raise RuntimeError("Cannot update with this serializer") + def create(self, validated_data): """Publish an RFC""" published = validated_data.pop("published") @@ -515,6 +504,182 @@ def _create_rfc(self, validated_data): return rfc +class EditableRfcSerializer(serializers.ModelSerializer): + # Would be nice to reconcile this with ietf.doc.serializers.RfcSerializer. + # The purposes of that serializer (representing data for Red) and this one + # (accepting updates from Purple) are different enough that separate formats + # may be needed, but if not it'd be nice to have a single RfcSerializer that + # can serve both. + # + # Should also consider whether this and RfcPubSerializer should merge. + # + # Treats published and subseries fields as write-only. This isn't quite correct, + # but makes it easier and we don't currently use the serialized value except for + # debugging. + published = serializers.DateTimeField( + default_timezone=datetime.timezone.utc, + write_only=True, + ) + authors = RfcAuthorSerializer(many=True, min_length=1, source="rfcauthor_set") + subseries = serializers.ListField( + child=SubseriesNameField(required=False), + write_only=True, + ) + + class Meta: + model = Document + fields = [ + "published", + "title", + "authors", + "stream", + "abstract", + "pages", + "std_level", + "subseries", + ] + + def create(self, validated_data): + raise RuntimeError("Cannot create with this serializer") + + def update(self, instance, validated_data): + assert isinstance(instance, Document) + assert instance.type_id == "rfc" + rfc = instance # get better name + + system_person = Person.objects.get(name="(System)") + + # Remove data that needs special handling. Use a singleton object to detect + # missing values in case we ever support a value that needs None as an option. + omitted = object() + published = validated_data.pop("published", omitted) + subseries = validated_data.pop("subseries", omitted) + authors_data = validated_data.pop("rfcauthor_set", omitted) + + # Transaction to clean up if something fails + with transaction.atomic(): + # update the rfc Document itself + rfc_changes = [] + rfc_events = [] + + for attr, new_value in validated_data.items(): + old_value = getattr(rfc, attr) + if new_value != old_value: + rfc_changes.append( + f"changed {attr} to '{new_value}' from '{old_value}'" + ) + setattr(rfc, attr, new_value) + if len(rfc_changes) > 0: + rfc_change_summary = f"{', '.join(rfc_changes)}" + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + by=system_person, + type="sync_from_rfc_editor", + desc=f"Changed metadata: {rfc_change_summary}", + ) + ) + if authors_data is not omitted: + rfc_events.extend(_update_authors(instance, authors_data)) + + if published is not omitted: + published_event = rfc.latest_event(type="published_rfc") + if published_event is None: + # unexpected, but possible in theory + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="published_rfc", + time=published, + by=system_person, + desc="RFC published", + ) + ) + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="sync_from_rfc_editor", + by=system_person, + desc=( + f"Set publication timestamp to {published.isoformat()}" + ), + ) + ) + else: + original_pub_time = published_event.time + if published != original_pub_time: + published_event.time = published + published_event.save() + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="sync_from_rfc_editor", + by=system_person, + desc=( + f"Changed publication time to " + f"{published.isoformat()} from " + f"{original_pub_time.isoformat()}" + ) + ) + ) + + # update subseries relations + if subseries is not omitted: + for subseries_doc_name in subseries: + ss_slug = subseries_doc_name[:3] + subseries_doc, ss_doc_created = Document.objects.get_or_create( + type_id=ss_slug, name=subseries_doc_name + ) + if ss_doc_created: + subseries_doc.docevent_set.create( + type=f"{ss_slug}_doc_created", + by=system_person, + desc=f"Created {subseries_doc_name} via update of {rfc.name}", + ) + _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create( + relationship_id="contains", target=rfc + ) + if ss_rel_created: + subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + rfc_events.append( + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + ) + # Delete subseries relations that are no longer current + stale_subseries_relations = rfc.relations_that("contains").exclude( + source__name__in=subseries + ) + for stale_relation in stale_subseries_relations: + stale_subseries_doc = stale_relation.source + rfc_events.append( + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Removed {rfc.name} from {stale_subseries_doc.name}", + ) + ) + stale_subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Removed {rfc.name} from {stale_subseries_doc.name}", + ) + stale_subseries_relations.delete() + if len(rfc_events) > 0: + rfc.save_with_history(rfc_events) + return rfc + + class RfcFileSerializer(serializers.Serializer): # The structure of this serializer is constrained by what openapi-generator-cli's # python generator can correctly serialize as multipart/form-data. It does not diff --git a/ietf/api/tests_serializers_rpc.py b/ietf/api/tests_serializers_rpc.py new file mode 100644 index 0000000000..1babb4c30f --- /dev/null +++ b/ietf/api/tests_serializers_rpc.py @@ -0,0 +1,139 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from django.utils import timezone + +from ietf.utils.test_utils import TestCase +from ietf.doc.models import Document +from ietf.doc.factories import WgRfcFactory +from .serializers_rpc import EditableRfcSerializer + + +class EditableRfcSerializerTests(TestCase): + def test_create(self): + serializer = EditableRfcSerializer( + data={ + "published": timezone.now(), + "title": "Yadda yadda yadda", + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + "stream": "ietf", + "abstract": "A long time ago in a galaxy far, far away...", + "pages": 3, + "std_level": "inf", + "subseries": ["fyi999"], + } + ) + self.assertTrue(serializer.is_valid()) + with self.assertRaises(RuntimeError, msg="serializer does not allow create()"): + serializer.save() + + def test_update(self): + rfc = WgRfcFactory(pages=10) + serializer = EditableRfcSerializer( + instance=rfc, + data={ + "published": timezone.now(), + "title": "Yadda yadda yadda", + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + "stream": "ise", + "abstract": "A long time ago in a galaxy far, far away...", + "pages": 3, + "std_level": "inf", + "subseries": ["fyi999"], + }, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(result.title, "Yadda yadda yadda") + self.assertEqual( + list( + result.rfcauthor_set.values( + "titlepage_name", "is_editor", "affiliation", "country" + ) + ), + [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + ) + self.assertEqual(result.stream_id, "ise") + self.assertEqual( + result.abstract, "A long time ago in a galaxy far, far away..." + ) + self.assertEqual(result.pages, 3) + self.assertEqual(result.std_level_id, "inf") + self.assertEqual( + result.part_of(), + [Document.objects.get(name="fyi999")], + ) + + def test_partial_update(self): + # We could test other permutations of fields, but authors is a partial update + # we know we are going to use, so verifying that one in particular. + rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan") + serializer = EditableRfcSerializer( + partial=True, + instance=rfc, + data={ + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + }, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(rfc.title, "padawan") + self.assertEqual( + list( + result.rfcauthor_set.values( + "titlepage_name", "is_editor", "affiliation", "country" + ) + ), + [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + ) + self.assertEqual(result.stream_id, "ietf") + self.assertEqual(result.abstract, "do or do not") + self.assertEqual(result.pages, 10) + self.assertEqual(result.std_level_id, "ps") + self.assertEqual(result.part_of(), []) + + # Test only a field on the Document itself to be sure that it works + serializer = EditableRfcSerializer( + partial=True, + instance=rfc, + data={"title": "jedi master"}, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(rfc.title, "jedi master") diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 9273590b28..8862bbf866 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -35,7 +35,7 @@ NotificationAckSerializer, RfcPubSerializer, RfcFileSerializer, EditableRfcSerializer, ) -from ietf.doc.models import Document, DocHistory, RfcAuthor +from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent from ietf.doc.serializers import RfcAuthorSerializer from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage from ietf.doc.tasks import signal_update_rfc_metadata_task @@ -279,6 +279,16 @@ class RfcViewSet(mixins.UpdateModelMixin, viewsets.GenericViewSet): lookup_field = "rfc_number" serializer_class = EditableRfcSerializer + def perform_update(self, serializer): + DocEvent.objects.create( + doc=serializer.instance, + rev=serializer.instance.rev, + by=Person.objects.get(name="(System)"), + type="sync_from_rfc_editor", + desc="Metadata update from RFC Editor", + ) + super().perform_update(serializer) + @action(detail=False, serializer_class=OriginalStreamSerializer) def rfc_original_stream(self, request): rfcs = self.get_queryset().annotate( diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index 36076c30be..a7ea640be8 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -27,6 +27,7 @@ class RfcAuthorSerializer(serializers.ModelSerializer): source="person.get_absolute_url", required=False, help_text="URL for person link (relative to datatracker base URL)", + read_only=True, ) class Meta: diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 42fab7d472..396b3fcfa4 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -740,14 +740,26 @@ def _rfcauthor_from_documentauthor(docauthor: DocumentAuthor) -> RfcAuthor: new_author.document = rfc new_author.order = order + 1 new_author.save() - changes.append(f'Added "{new_author.titlepage_name}" as author') + if new_author.person_id is not None: + person_desc = f"Person {new_author.person_id}" + else: + person_desc = "no Person linked" + changes.append( + f'Added "{new_author.titlepage_name}" ({person_desc}) as author' + ) # Any authors left in original_authors are no longer in the list, so remove them for removed_author in original_authors: # Skip actual removal of old authors if we are converting from the # DocumentAuthor models - the original_authors were just stand-ins anyway. if not converting_from_docauthors: removed_author.delete() - changes.append(f'Removed "{removed_author.titlepage_name}" as author') + if removed_author.person_id is not None: + person_desc = f"Person {removed_author.person_id}" + else: + person_desc = "no Person linked" + changes.append( + f'Removed "{removed_author.titlepage_name}" ({person_desc}) as author' + ) # Create DocEvents, but leave it up to caller to save if by is None: by = Person.objects.get(name="(System)") From 481054511b9f07a47c41f854105e00616e61d3e2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 3 Mar 2026 11:36:45 -0400 Subject: [PATCH 080/158] feat: add area to FullDraftSerializer (#10487) * refactor: Document.area() + serializer * feat: add area to FullDraftSerializer --- ietf/api/serializers_rpc.py | 3 +++ ietf/doc/models.py | 16 ++++++++++++++++ ietf/doc/serializers.py | 19 +------------------ 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index d5f5363990..e51b917be4 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -27,6 +27,7 @@ update_rfcauthors, ) from ietf.group.models import Group +from ietf.group.serializers import AreaSerializer from ietf.name.models import StreamName, StdLevelName from ietf.person.models import Person from ietf.utils import log @@ -115,6 +116,7 @@ class FullDraftSerializer(serializers.ModelSerializer): name = serializers.CharField(max_length=255) title = serializers.CharField(max_length=255) group = serializers.SlugRelatedField(slug_field="acronym", read_only=True) + area = AreaSerializer(read_only=True) # Other fields we need to add / adjust source_format = serializers.SerializerMethodField() @@ -133,6 +135,7 @@ class Meta: "stream", "title", "group", + "area", "abstract", "pages", "source_format", diff --git a/ietf/doc/models.py b/ietf/doc/models.py index cc28951be0..f1b319367e 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1147,6 +1147,22 @@ def request_closed_time(self, review_req): e = self.latest_event(ReviewRequestDocEvent, type="closed_review_request", review_request=review_req) return e.time if e and e.time else None + @property + def area(self) -> Group | None: + """Get area for document, if one exists + + None for non-IETF-stream documents. N.b., this is stricter than Group.area() and + uses different logic from Document.area_acronym(). + """ + if self.stream_id != "ietf": + return None + if self.group is None: + return None + parent = self.group.parent + if parent.type_id == "area": + return parent + return None + def area_acronym(self): g = self.group if g: diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index a7ea640be8..139ae9aa7e 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -230,7 +230,7 @@ class RfcMetadataSerializer(serializers.ModelSerializer): status = RfcStatusSerializer(source="*") authors = serializers.SerializerMethodField() group = GroupSerializer() - area = serializers.SerializerMethodField() + area = AreaSerializer(read_only=True) stream = StreamNameSerializer() ad = AreaDirectorSerializer(read_only=True, allow_null=True) group_list_email = serializers.EmailField(source="group.list_email", read_only=True) @@ -288,23 +288,6 @@ def get_authors(self, doc: Document): many=True, ).data - @extend_schema_field(AreaSerializer(required=False)) - def get_area(self, doc: Document): - """Get area for the RFC - - This logic might be better moved to Document or a combination of Document - and Group. The current (2026-02-24) Group.area() method is not strict enough: - it does not limit to WG groups or IETF-stream documents. - """ - if doc.stream_id != "ietf": - return None - if doc.group is None: - return None - parent = doc.group.parent - if parent.type_id == "area": - return AreaSerializer(parent).data - return None - @extend_schema_field(DocIdentifierSerializer(many=True)) def get_identifiers(self, doc: Document): identifiers = [] From 47d3734955071d1ccc54787698e751c74ce4d303 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Mar 2026 12:32:36 -0400 Subject: [PATCH 081/158] chore(deps): bump types-pytz from 2025.2.0.20250809 to 2025.2.0.20251108 (#10424) Bumps [types-pytz](https://github.com/typeshed-internal/stub_uploader) from 2025.2.0.20250809 to 2025.2.0.20251108. - [Commits](https://github.com/typeshed-internal/stub_uploader/commits) --- updated-dependencies: - dependency-name: types-pytz dependency-version: 2025.2.0.20251108 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cb583d5dc9..3d54b104ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -74,7 +74,7 @@ python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=2.0.0 # from TastyPie pytz==2025.2 # Pinned as changes need to be vetted for their effect on Meeting fields -types-pytz==2025.2.0.20250809 # match pytz version +types-pytz==2025.2.0.20251108 # match pytz version requests>=2.32.4 types-requests>=2.32.4 requests-mock>=1.12.1 From 1799245dc6ce82301b0790412957ccfa19910dc1 Mon Sep 17 00:00:00 2001 From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:45:46 +0000 Subject: [PATCH 082/158] ci: update base image target version to 20260304T1633 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 71370fabee..ce1828052e 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260211T1901 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260304T1633 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 947f3790e4..6be54fb6b0 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260211T1901 +20260304T1633 From 7f28542c82e2c51210daf77ca10f9682c0ea709d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 9 Mar 2026 14:02:57 -0300 Subject: [PATCH 083/158] fix: finish dropping email as RfcAuthor field (#10512) * fix: fix admin / Document.author_list() * fix: update RfcAuthorResource email is still accessible, but read only * fix: admin search by RfcAuthor email --- ietf/doc/admin.py | 4 +++- ietf/doc/models.py | 16 +++++++++++----- ietf/doc/resources.py | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index b604d4f096..0d04e8db3a 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -241,7 +241,9 @@ def is_deleted(self, instance): admin.site.register(StoredObject, StoredObjectAdmin) class RfcAuthorAdmin(admin.ModelAdmin): + # the email field in the list_display/readonly_fields works through a @property list_display = ['id', 'document', 'titlepage_name', 'person', 'email', 'affiliation', 'country', 'order'] - search_fields = ['document__name', 'titlepage_name', 'person__name', 'email', 'affiliation', 'country'] + search_fields = ['document__name', 'titlepage_name', 'person__name', 'person__email__address', 'affiliation', 'country'] raw_id_fields = ["document", "person"] + readonly_fields = ["email"] admin.site.register(RfcAuthor, RfcAuthorAdmin) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index f1b319367e..868bc4ac47 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -466,11 +466,12 @@ def author_persons(self): def author_list(self): """List of author emails""" - author_qs = ( - self.rfcauthor_set - if self.type_id == "rfc" and self.rfcauthor_set.exists() - else self.documentauthor_set - ).select_related("email").order_by("order") + if self.type_id == "rfc" and self.rfcauthor_set.exists(): + author_qs = self.rfcauthor_set.select_related("person").order_by("order") + else: + author_qs = self.documentauthor_set.select_related("email").order_by( + "order" + ) best_addresses = [] for author in author_qs: if author.email: @@ -953,6 +954,11 @@ class Meta: @property def email(self) -> Email | None: return self.person.email() if self.person else None + + def format_for_titlepage(self): + if self.is_editor: + return f"{self.titlepage_name}, Ed." + return self.titlepage_name class DocumentAuthorInfo(models.Model): diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index 556465a522..1d86df78d0 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -897,7 +897,7 @@ class Meta: class RfcAuthorResource(ModelResource): document = ToOneField(DocumentResource, 'document') person = ToOneField(PersonResource, 'person', null=True) - email = ToOneField(EmailResource, 'email', null=True) + email = ToOneField(EmailResource, 'email', null=True, readonly=True) class Meta: queryset = RfcAuthor.objects.all() serializer = api.Serializer() From 809e7682db30279cb715f47c89ae546e320c9c76 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 11 Mar 2026 13:07:27 -0500 Subject: [PATCH 084/158] chore: remove task explorer from devcontainer (#10532) --- .devcontainer/devcontainer.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2cfff78853..e4964e8909 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -32,7 +32,6 @@ "mutantdino.resourcemonitor", "oderwat.indent-rainbow", "redhat.vscode-yaml", - "spmeesseman.vscode-taskexplorer", "ms-python.pylint", "charliermarsh.ruff" ], From d4a594ddd4a9dd0bd575465748627f7fea68aac3 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 11 Mar 2026 15:12:22 -0300 Subject: [PATCH 085/158] feat: rfc-index generation (#10526) * chore: stand-in red_bucket STORAGE for dev * feat: rfc index text generation (WIP) Text generation works. Also includes XML generation that is not yet converted. Based on Kesara's implementations in the purple repo. * feat: rfc index XML generation (WIP) * feat: Document.keywords + migration * feat: keywords API * feat: keywords in rfc-index.xml * fix: better stream/area/wg_acronym Still some disagreements, not sure if that's data or logic driven * fix: NON WORKING GROUP logic May need more attention * fix: add rev to draft name * fix: interleave unpublished RFC records * fix: lint * refactor: use lxml * fix: multi-paragraph abstracts * feat: RFCINDEX_MATCH_LEGACY_XML option * fix: zero pad DOIs * fix: better NON WORKING GROUP id * fix: reorder elements * refactor: extract repeated code * refactor: unify DOI generation * fix: modern DOI proxy URL for ATOM feed * refactor: settings.RFC_EDITOR_ERRATA_BASE_URL Drop unused settings.RFC_EDITOR_ERRATA_URL * chore: real red_bucket storage cfg * fix: handle missing json for prod/dev/test * chore: straighten out S3 saving * chore(dev): FileSystemStorage for red_bucket dev (commented out) * chore: configurable bucket path for JSON inputs * test: tests_rfcindex.py Not great coverage, but exercises the generators a bit. * fix: lint + consistent var naming * test: improve test coverage / testability * fix: lint --- docker/configs/settings_local.py | 11 + ietf/api/serializers_rpc.py | 2 + ietf/doc/api.py | 6 - ietf/doc/factories.py | 6 + ietf/doc/feeds.py | 9 +- ...3_dochistory_keywords_document_keywords.py | 31 ++ ietf/doc/models.py | 21 + ietf/doc/serializers.py | 4 +- ietf/doc/views_doc.py | 4 +- ietf/settings.py | 10 +- ietf/settings_test.py | 8 +- ietf/sync/rfcindex.py | 480 ++++++++++++++++++ ietf/sync/tests_rfcindex.py | 230 +++++++++ ietf/templates/sync/rfc-index.txt | 69 +++ k8s/settings_local.py | 33 ++ 15 files changed, 907 insertions(+), 17 deletions(-) create mode 100644 ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py create mode 100644 ietf/sync/rfcindex.py create mode 100644 ietf/sync/tests_rfcindex.py create mode 100644 ietf/templates/sync/rfc-index.txt diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 1d4e6916b9..94adc516a4 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -101,6 +101,17 @@ ), } +# For dev on rfc-index generation, create a red_bucket/ directory in the project root +# and uncomment these settings. Generated files will appear in this directory. To +# generate an accurate index, put up-to-date copies of unusable-rfc-numbers.json, +# april-first-rfc-numbers.json, and publication-std-levels.json in this directory +# before generating the index. +# +# STORAGES["red_bucket"] = { +# "BACKEND": "django.core.files.storage.FileSystemStorage", +# "OPTIONS": {"location": "red_bucket"}, +# } + APP_API_TOKENS = { "ietf.api.red_api" : ["devtoken", "redtoken"], # Not a real secret "ietf.api.views_rpc" : ["devtoken"], # Not a real secret diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index e51b917be4..c17cbc64ce 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -300,6 +300,7 @@ class Meta: "obsoletes", "updates", "subseries", + "keywords", ] def validate(self, data): @@ -540,6 +541,7 @@ class Meta: "pages", "std_level", "subseries", + "keywords", ] def create(self, validated_data): diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 75993f463e..73fff6b27f 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -4,13 +4,11 @@ from django.db.models import ( BooleanField, Count, - JSONField, OuterRef, Prefetch, Q, QuerySet, Subquery, - Value, ) from django.db.models.functions import TruncDate from django_filters import rest_framework as filters @@ -160,10 +158,6 @@ def augment_rfc_queryset(queryset: QuerySet[Document]): output_field=BooleanField(), ) ) - .annotate( - # TODO implement this fake field for real - keywords=Value(["keyword"], output_field=JSONField()), - ) ) diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index bc38765446..1a178c6f31 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -311,6 +311,12 @@ class Meta: def desc(self): return 'New version available %s-%s'%(self.doc.name,self.rev) +class PublishedRfcDocEventFactory(DocEventFactory): + class Meta: + model = DocEvent + type = "published_rfc" + doc = factory.SubFactory(WgRfcFactory) + class StateDocEventFactory(DocEventFactory): class Meta: model = StateDocEvent diff --git a/ietf/doc/feeds.py b/ietf/doc/feeds.py index 500ed3cb18..afe96cf0df 100644 --- a/ietf/doc/feeds.py +++ b/ietf/doc/feeds.py @@ -1,5 +1,4 @@ -# Copyright The IETF Trust 2007-2020, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2007-2026, All Rights Reserved import debug # pyflakes:ignore @@ -263,9 +262,11 @@ def item_extra_kwargs(self, item): ) extra.update({"media_contents": media_contents}) - extra.update({"doi": "10.17487/%s" % item.name.upper()}) extra.update( - {"doiuri": "http://dx.doi.org/10.17487/%s" % item.name.upper()} + { + "doi": item.doi, + "doiuri": f"https://doi.org/{item.doi}", + } ) # R104 Publisher (Mandatory - but we need a string from them first) diff --git a/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py b/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py new file mode 100644 index 0000000000..5e2513e15a --- /dev/null +++ b/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models +import ietf.doc.models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0032_remove_rfcauthor_email"), + ] + + operations = [ + migrations.AddField( + model_name="dochistory", + name="keywords", + field=models.JSONField( + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + migrations.AddField( + model_name="document", + name="keywords", + field=models.JSONField( + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 868bc4ac47..7b23a62c45 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -13,6 +13,7 @@ from io import BufferedReader from pathlib import Path +from django.core.exceptions import ValidationError from django.db.models import Q from lxml import etree from typing import Optional, Protocol, TYPE_CHECKING, Union @@ -109,6 +110,15 @@ class Meta: IESG_STATCHG_CONFLREV_ACTIVE_STATES = ("iesgeval", "defer") IESG_SUBSTATE_TAGS = ('ad-f-up', 'need-rev', 'extpty') + +def validate_doc_keywords(value): + if ( + not isinstance(value, list | tuple | set) + or not all(isinstance(elt, str) for elt in value) + ): + raise ValidationError("Value must be an array of strings") + + class DocumentInfo(models.Model): """Any kind of document. Draft, RFC, Charter, IPR Statement, Liaison Statement""" time = models.DateTimeField(default=timezone.now) # should probably have auto_now=True @@ -142,6 +152,17 @@ class DocumentInfo(models.Model): uploaded_filename = models.TextField(blank=True) note = models.TextField(blank=True) rfc_number = models.PositiveIntegerField(blank=True, null=True) # only valid for type="rfc" + keywords = models.JSONField( + default=list, + max_length=1000, + validators=[validate_doc_keywords], + ) + + @property + def doi(self) -> str | None: + if self.type_id == "rfc" and self.rfc_number is not None: + return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number:04d}" + return None def file_extension(self): if not hasattr(self, '_cached_extension'): diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index 139ae9aa7e..3651670962 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -291,9 +291,9 @@ def get_authors(self, doc: Document): @extend_schema_field(DocIdentifierSerializer(many=True)) def get_identifiers(self, doc: Document): identifiers = [] - if doc.rfc_number: + if doc.doi: identifiers.append( - DocIdentifier(type="doi", value=f"10.17487/RFC{doc.rfc_number:04d}") + DocIdentifier(type="doi", value=doc.doi) ) return DocIdentifierSerializer(instance=identifiers, many=True).data diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 0ae7520681..c1f6352ac3 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -1285,9 +1285,7 @@ def document_bibtex(request, name, rev=None): break elif doc.type_id == "rfc": - # This needs to be replaced with a lookup, as the mapping may change - # over time. - doi = f"10.17487/RFC{doc.rfc_number:04d}" + doi = doc.doi if doc.is_dochistory(): latest_event = doc.latest_event(type='new_revision', rev=rev) diff --git a/ietf/settings.py b/ietf/settings.py index 71b110d762..e0b4f20118 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -838,6 +838,11 @@ def skip_unreadable_post(record): "slides", ] +# Other storages +STORAGES["red_bucket"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "red_bucket"}, +} # Override this in settings_local.py if needed # *_PATH variables ends with a slash/ . @@ -932,10 +937,11 @@ def skip_unreadable_post(record): RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" RFC_EDITOR_ERRATA_JSON_URL = "https://www.rfc-editor.org/errata.json" -RFC_EDITOR_ERRATA_URL = "https://www.rfc-editor.org/errata_search.php?rfc={rfc_number}" RFC_EDITOR_INLINE_ERRATA_URL = "https://www.rfc-editor.org/rfc/inline-errata/rfc{rfc_number}.html" +RFC_EDITOR_ERRATA_BASE_URL = "https://www.rfc-editor.org/errata/" RFC_EDITOR_INFO_BASE_URL = "https://www.rfc-editor.org/info/" + # NomCom Tool settings ROLODEX_URL = "" NOMCOM_PUBLIC_KEYS_DIR = '/a/www/nomcom/public_keys/' @@ -1570,3 +1576,5 @@ def skip_unreadable_post(record): YOUTUBE_DOMAINS = ['www.youtube.com', 'youtube.com', 'youtu.be', 'm.youtube.com', 'youtube-nocookie.com', 'www.youtube-nocookie.com'] + +IETF_DOI_PREFIX = "10.17487" diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 6479069db0..1f5a7e8ddc 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -14,7 +14,7 @@ import shutil import tempfile from ietf.settings import * # pyflakes:ignore -from ietf.settings import ORIG_AUTH_PASSWORD_VALIDATORS +from ietf.settings import ORIG_AUTH_PASSWORD_VALIDATORS, STORAGES import debug # pyflakes:ignore debug.debug = True @@ -114,3 +114,9 @@ def tempdir_with_cleanup(**kwargs): AUTH_PASSWORD_VALIDATORS = ORIG_AUTH_PASSWORD_VALIDATORS except NameError: pass + +# Use InMemoryStorage for red bucket storage +STORAGES["red_bucket"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "red_bucket"}, +} diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py new file mode 100644 index 0000000000..b15846094f --- /dev/null +++ b/ietf/sync/rfcindex.py @@ -0,0 +1,480 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import json +from collections import defaultdict +from collections.abc import Container +from dataclasses import dataclass +from io import StringIO, BytesIO +from itertools import chain +from operator import attrgetter, itemgetter +from pathlib import Path +from textwrap import fill +from urllib.parse import urljoin + +from django.conf import settings +from lxml import etree + +from django.core.files.storage import storages +from django.db import models +from django.db.models.functions import Substr, Cast +from django.template.loader import render_to_string +from django.utils import timezone + +from ietf.doc.models import Document +from ietf.name.models import StdLevelName +from ietf.utils.log import log + +FORMATS_FOR_INDEX = ["txt", "html", "pdf", "xml", "ps"] + + +def format_rfc_number(n): + """Format an RFC number (or subseries doc number) + + Set settings.RFCINDEX_MATCH_LEGACY_XML=True for the legacy (leading-zero) format. + That is for debugging only - tests will fail. + """ + if getattr(settings, "RFCINDEX_MATCH_LEGACY_XML", False): + return format(n, "04") + else: + return format(n) + + +def errata_url(rfc: Document): + return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}") + + +def save_to_red_bucket(filename: str, content: BytesIO | StringIO): + red_bucket = storages["red_bucket"] + bucket_path = str(Path(getattr(settings, "RFCINDEX_OUTPUT_PATH", "")) / filename) + if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True): + # Django 4.2's FileSystemStorage does not support allow_overwrite. + red_bucket.delete(bucket_path) + red_bucket.save(bucket_path, content) + log(f"Saved {bucket_path} in red_bucket storage") + + +@dataclass +class UnusableRfcNumber: + rfc_number: int + comment: str + + +def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: + FILENAME = "unusable-rfc-numbers.json" + bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + return [] # pragma: no cover + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE == "development": + return [] # pragma: no cover + raise + assert all(isinstance(record["number"], int) for record in records) + assert all(isinstance(record["comment"], str) for record in records) + return [ + UnusableRfcNumber(rfc_number=record["number"], comment=record["comment"]) + for record in sorted(records, key=itemgetter("number")) + ] + + +def get_april1_rfc_numbers() -> Container[int]: + FILENAME = "april-first-rfc-numbers.json" + bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + return [] # pragma: no cover + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE == "development": + return [] # pragma: no cover + raise + assert all(isinstance(record, int) for record in records) + return records + + +def get_publication_std_levels() -> dict[int, StdLevelName]: + FILENAME = "publication-std-levels.json" + bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + values: dict[int, StdLevelName] = {} + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + # intentionally fall through instead of return here + else: + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE != "development": + raise + else: + assert all(isinstance(record["number"], int) for record in records) + values = { + record["number"]: StdLevelName.objects.get( + slug=record["publication_std_level"] + ) + for record in records + } + # defaultdict to return "unknown" for any missing values + unknown_std_level = StdLevelName.objects.get(slug="unkn") + return defaultdict(lambda: unknown_std_level, values) + + +def format_ordering(rfc_number): + if rfc_number < 8650: + ordering = ["txt", "ps", "pdf", "html", "xml"] + else: + ordering = ["html", "txt", "ps", "pdf", "xml"] + return ordering.index # return the method + + +def get_rfc_text_index_entries(): + """Returns RFC entries for rfc-index.txt""" + entries = [] + april1_rfc_numbers = get_april1_rfc_numbers() + published_rfcs = Document.objects.filter(type_id="rfc").order_by("rfc_number") + rfcs = sorted( + chain(published_rfcs, get_unusable_rfc_numbers()), key=attrgetter("rfc_number") + ) + for rfc in rfcs: + if isinstance(rfc, UnusableRfcNumber): + entries.append(f"{format_rfc_number(rfc.rfc_number)} Not Issued.") + else: + assert isinstance(rfc, Document) + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + published_at = rfc.pub_date() + date = ( + published_at.strftime("1 %B %Y") + if rfc.rfc_number in april1_rfc_numbers + else published_at.strftime("%B %Y") + ) + + # formats + formats = ", ".join( + sorted( + [ + format["fmt"] + for format in rfc.formats() + if format["fmt"] in FORMATS_FOR_INDEX + ], + key=format_ordering(rfc.rfc_number), + ) + ).upper() + + # obsoletes + obsoletes = "" + obsoletes_documents = sorted( + rfc.related_that_doc("obs"), + key=attrgetter("rfc_number"), + ) + if len(obsoletes_documents) > 0: + obsoletes_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in obsoletes_documents + ) + obsoletes = f" (Obsoletes {obsoletes_names})" + + # obsoleted by + obsoleted_by = "" + obsoleted_by_documents = sorted( + rfc.related_that("obs"), + key=attrgetter("rfc_number"), + ) + if len(obsoleted_by_documents) > 0: + obsoleted_by_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in obsoleted_by_documents + ) + obsoleted_by = f" (Obsoleted by {obsoleted_by_names})" + + # updates + updates = "" + updates_documents = sorted( + rfc.related_that_doc("updates"), + key=attrgetter("rfc_number"), + ) + if len(updates_documents) > 0: + updates_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in updates_documents + ) + updates = f" (Updates {updates_names})" + + # updated by + updated_by = "" + updated_by_documents = sorted( + rfc.related_that("updates"), + key=attrgetter("rfc_number"), + ) + if len(updated_by_documents) > 0: + updated_by_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in updated_by_documents + ) + updated_by = f" (Updated by {updated_by_names})" + + doc_relations = f"{obsoletes}{obsoleted_by}{updates}{updated_by} " + + # subseries + subseries = ",".join( + f"{container.type.slug}{format_rfc_number(int(container.name[3:]))}" + for container in rfc.part_of() + ).upper() + if subseries: + subseries = f"(Also {subseries}) " + + entry = fill( + ( + f"{format_rfc_number(rfc.rfc_number)} {rfc.title}. {authors}. {date}. " + f"(Format: {formats}){doc_relations}{subseries}" + f"(Status: {str(rfc.std_level).upper()}) " + f"(DOI: {rfc.doi})" + ), + width=73, + subsequent_indent=" " * 5, + ) + entries.append(entry) + + return entries + + +def add_subseries_xml_index_entries(rfc_index, ss_type, include_all=False): + """Add subseries entries for rfc-index.xml""" + # subseries docs annotated with numeric number + ss_docs = list( + Document.objects.filter(type_id=ss_type) + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + ) + if len(ss_docs) == 0: + return # very much not expected + highest_number = ss_docs[0].number + for ss_number in range(1, highest_number + 1): + if ss_docs[-1].number == ss_number: + this_ss_doc = ss_docs.pop() + contained_rfcs = this_ss_doc.contains() + else: + contained_rfcs = [] + if len(contained_rfcs) == 0 and not include_all: + continue + entry = etree.SubElement(rfc_index, f"{ss_type}-entry") + etree.SubElement( + entry, "doc-id" + ).text = f"{ss_type.upper()}{format_rfc_number(ss_number)}" + if len(contained_rfcs) > 0: + is_also = etree.SubElement(entry, "is-also") + for rfc in sorted(contained_rfcs, key=attrgetter("rfc_number")): + etree.SubElement( + is_also, "doc-id" + ).text = f"RFC{format_rfc_number(rfc.rfc_number)}" + + +def add_related_xml_index_entries(root: etree.Element, rfc: Document, tag: str): + relation_getter = { + "obsoletes": lambda doc: doc.related_that_doc("obs"), + "obsoleted-by": lambda doc: doc.related_that("obs"), + "updates": lambda doc: doc.related_that_doc("updates"), + "updated-by": lambda doc: doc.related_that("updates"), + } + related_docs = sorted( + relation_getter[tag](rfc), + key=attrgetter("rfc_number"), + ) + if len(related_docs) > 0: + element = etree.SubElement(root, tag) + for doc in related_docs: + etree.SubElement( + element, "doc-id" + ).text = f"RFC{format_rfc_number(doc.rfc_number)}" + + +def add_rfc_xml_index_entries(rfc_index): + """Add RFC entries for rfc-index.xml""" + entries = [] + april1_rfc_numbers = get_april1_rfc_numbers() + publication_statuses = get_publication_std_levels() + + published_rfcs = Document.objects.filter(type_id="rfc").order_by("rfc_number") + + # Iterators for unpublished and published, both sorted by number + unpublished_iter = iter(get_unusable_rfc_numbers()) + published_iter = iter(published_rfcs) + + # Prime the next_* values + next_unpublished = next(unpublished_iter, None) + next_published = next(published_iter, None) + + while next_published is not None or next_unpublished is not None: + if next_unpublished is not None and ( + next_published is None + or next_unpublished.rfc_number < next_published.rfc_number + ): + entry = etree.SubElement(rfc_index, "rfc-not-issued-entry") + etree.SubElement( + entry, "doc-id" + ).text = f"RFC{format_rfc_number(next_unpublished.rfc_number)}" + entries.append(entry) + next_unpublished = next(unpublished_iter, None) + continue + + rfc = next_published # hang on to this + next_published = next(published_iter, None) # prep for next iteration + entry = etree.SubElement(rfc_index, "rfc-entry") + + etree.SubElement( + entry, "doc-id" + ).text = f"RFC{format_rfc_number(rfc.rfc_number)}" + etree.SubElement(entry, "title").text = rfc.title + + for author in rfc.rfcauthor_set.all(): + author_element = etree.SubElement(entry, "author") + etree.SubElement(author_element, "name").text = author.titlepage_name + if author.is_editor: + etree.SubElement(author_element, "title").text = "Editor" + + date = etree.SubElement(entry, "date") + published_at = rfc.pub_date() + etree.SubElement(date, "month").text = published_at.strftime("%B") + if rfc.rfc_number in april1_rfc_numbers: + etree.SubElement(date, "day").text = str(published_at.day) + etree.SubElement(date, "year").text = str(published_at.year) + + format_ = etree.SubElement(entry, "format") + fmts = [ff["fmt"] for ff in rfc.formats() if ff["fmt"] in FORMATS_FOR_INDEX] + for fmt in sorted(fmts, key=format_ordering(rfc.rfc_number)): + match_legacy = getattr(settings, "RFCINDEX_MATCH_LEGACY_XML", False) + etree.SubElement(format_, "file-format").text = ( + "ASCII" if match_legacy and fmt == "txt" else fmt.upper() + ) + + etree.SubElement(entry, "page-count").text = str(rfc.pages) + + if len(rfc.keywords) > 0: + keywords = etree.SubElement(entry, "keywords") + for keyword in rfc.keywords: + etree.SubElement(keywords, "kw").text = keyword.strip() + + if rfc.abstract: + abstract = etree.SubElement(entry, "abstract") + for paragraph in rfc.abstract.split("\n\n"): + etree.SubElement(abstract, "p").text = paragraph.strip() + + draft = rfc.came_from_draft() + if draft is not None: + etree.SubElement(entry, "draft").text = f"{draft.name}-{draft.rev}" + + part_of_documents = rfc.part_of() + if len(part_of_documents) > 0: + is_also = etree.SubElement(entry, "is-also") + for doc in part_of_documents: + etree.SubElement(is_also, "doc-id").text = doc.name.upper() + + add_related_xml_index_entries(entry, rfc, "obsoletes") + add_related_xml_index_entries(entry, rfc, "obsoleted-by") + add_related_xml_index_entries(entry, rfc, "updates") + add_related_xml_index_entries(entry, rfc, "updated-by") + + etree.SubElement(entry, "current-status").text = rfc.std_level.name.upper() + etree.SubElement(entry, "publication-status").text = publication_statuses[ + rfc.rfc_number + ].name.upper() + etree.SubElement(entry, "stream").text = ( + "INDEPENDENT" if rfc.stream_id == "ise" else rfc.stream.name + ) + + # Add area / wg_acronym + if rfc.stream_id == "ietf": + if rfc.group.type_id in ["individ", "area"]: + etree.SubElement(entry, "wg_acronym").text = "NON WORKING GROUP" + else: + if rfc.area is not None: + etree.SubElement(entry, "area").text = rfc.area.acronym + if rfc.group: + etree.SubElement(entry, "wg_acronym").text = rfc.group.acronym + + if rfc.tags.filter(slug="errata").exists(): + etree.SubElement(entry, "errata-url").text = errata_url(rfc) + etree.SubElement(entry, "doi").text = rfc.doi + entries.append(entry) + + +def create_rfc_txt_index(): + """Create text index of published documents""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating rfc-index.txt") + index = render_to_string( + "sync/rfc-index.txt", + { + "created_on": created_on, + "rfcs": get_rfc_text_index_entries(), + }, + ) + save_to_red_bucket("rfc-index.txt", StringIO(index)) + + +def create_rfc_xml_index(): + """Create XML index of published documents""" + XSI_NAMESPACE = "http://www.w3.org/2001/XMLSchema-instance" + XSI = "{" + XSI_NAMESPACE + "}" + + log("Creating rfc-index.xml") + rfc_index = etree.Element( + "rfc-index", + nsmap={ + None: "https://www.rfc-editor.org/rfc-index", + "xsi": XSI_NAMESPACE, + }, + attrib={ + XSI + "schemaLocation": ( + "https://www.rfc-editor.org/rfc-index " + "https://www.rfc-editor.org/rfc-index.xsd" + ), + }, + ) + + # add data + add_subseries_xml_index_entries(rfc_index, "bcp", include_all=True) + add_subseries_xml_index_entries(rfc_index, "fyi") + add_rfc_xml_index_entries(rfc_index) + add_subseries_xml_index_entries(rfc_index, "std") + + # make it pretty + pretty_index = etree.tostring( + rfc_index, + encoding="utf-8", + xml_declaration=True, + pretty_print=4, + ) + save_to_red_bucket("rfc-index.xml", BytesIO(pretty_index)) diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py new file mode 100644 index 0000000000..b0a8712fe1 --- /dev/null +++ b/ietf/sync/tests_rfcindex.py @@ -0,0 +1,230 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import json +from io import BytesIO, StringIO +from unittest import mock + +from django.core.files.storage import storages +from django.test.utils import override_settings +from lxml import etree + +from ietf.doc.factories import PublishedRfcDocEventFactory, IndividualRfcFactory +from ietf.name.models import DocTagName +from ietf.sync.rfcindex import ( + create_rfc_txt_index, + create_rfc_xml_index, + format_rfc_number, + save_to_red_bucket, get_unusable_rfc_numbers, get_april1_rfc_numbers, + get_publication_std_levels, +) +from ietf.utils.test_utils import TestCase + + +class RfcIndexTests(TestCase): + """Tests of rfc-index generation + + Tests are limited and should cover more cases. Needs: + * test of subseries docs + * test of related docs (obsoletes/updates + reverse directions) + * more thorough validation of index contents + + Be careful when calling create_rfc_txt_index() or create_rfc_xml_index(). These + will save to a storage by default, which can introduce cross-talk between tests. + Best to patch that method with a mock. + """ + + def setUp(self): + super().setUp() + red_bucket = storages["red_bucket"] + + # Create an unused RFC number + red_bucket.save( + "input/unusable-rfc-numbers.json", + StringIO(json.dumps([{"number": 123, "comment": ""}])), + ) + + # actual April 1 RFC + self.april_fools_rfc = PublishedRfcDocEventFactory( + time="2020-04-01T12:00:00Z", + doc=IndividualRfcFactory( + name="rfc4560", + rfc_number=4560, + stream_id="ise", + std_level_id="inf", + ), + ).doc + # Set up a JSON file to flag the April 1 RFC + red_bucket.save( + "input/april-first-rfc-numbers.json", + StringIO(json.dumps([self.april_fools_rfc.rfc_number])), + ) + + # non-April Fools RFC that happens to have been published on April 1 + self.rfc = PublishedRfcDocEventFactory( + time="2021-04-01T12:00:00Z", + doc__name="rfc10000", + doc__rfc_number=10000, + doc__std_level_id="std", + ).doc + self.rfc.tags.add(DocTagName.objects.get(slug="errata")) + + # Set up a publication-std-levels.json file to indicate the publication + # standard of self.rfc as different from its current value + red_bucket.save( + "input/publication-std-levels.json", + StringIO( + json.dumps( + [{"number": self.rfc.rfc_number, "publication_std_level": "ps"}] + ) + ), + ) + + def tearDown(self): + red_bucket = storages["red_bucket"] + red_bucket.delete("input/unusable-rfc-numbers.json") + red_bucket.delete("input/april-first-rfc-numbers.json") + red_bucket.delete("input/publication-std-levels.json") + super().tearDown() + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_rfc_txt_index(self, mock_save): + create_rfc_txt_index() + self.assertEqual(mock_save.call_count, 1) + self.assertEqual(mock_save.call_args[0][0], "rfc-index.txt") + contents = mock_save.call_args[0][1].read() + self.assertIn( + "123 Not Issued.", + contents, + ) + # No zero prefix! + self.assertNotIn( + "0123 Not Issued.", + contents, + ) + self.assertIn( + f"{self.april_fools_rfc.rfc_number} {self.april_fools_rfc.title}", + contents, + ) + self.assertIn("1 April 2020", contents) # from the April 1 RFC + self.assertIn( + f"{self.rfc.rfc_number} {self.rfc.title}", + contents, + ) + self.assertIn("April 2021", contents) # from the non-April 1 RFC + self.assertNotIn("1 April 2021", contents) + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_rfc_xml_index(self, mock_save): + create_rfc_xml_index() + self.assertEqual(mock_save.call_count, 1) + self.assertEqual(mock_save.call_args[0][0], "rfc-index.xml") + contents = mock_save.call_args[0][1].read() + ns = "{https://www.rfc-editor.org/rfc-index}" # NOT an f-string + index = etree.fromstring(contents) + + # We can aspire to validating the schema - currently does not conform because + # XSD expects 4-digit RFC numbers (etc). + # + # xmlschema = etree.XMLSchema(etree.fromstring( + # Path(__file__).with_name("rfc-index.xsd").read_bytes()) + # ) + # xmlschema.assertValid(index) + + children = list(index) # elements as list + # Should be one rfc-not-issued-entry + self.assertEqual(len(children), 3) + self.assertEqual( + [ + c.find(f"{ns}doc-id").text + for c in children + if c.tag == f"{ns}rfc-not-issued-entry" + ], + ["RFC123"], + ) + # Should be two rfc-entries + rfc_entries = { + c.find(f"{ns}doc-id").text: c for c in children if c.tag == f"{ns}rfc-entry" + } + + # Check the April Fool's entry + april_fools_entry = rfc_entries[self.april_fools_rfc.name.upper()] + self.assertEqual( + april_fools_entry.find(f"{ns}title").text, + self.april_fools_rfc.title, + ) + self.assertEqual( + [(c.tag, c.text) for c in april_fools_entry.find(f"{ns}date")], + [(f"{ns}month", "April"), (f"{ns}day", "1"), (f"{ns}year", "2020")], + ) + self.assertEqual( + april_fools_entry.find(f"{ns}current-status").text, + "INFORMATIONAL", + ) + self.assertEqual( + april_fools_entry.find(f"{ns}publication-status").text, + "UNKNOWN", + ) + + # Check the Regular entry + rfc_entry = rfc_entries[self.rfc.name.upper()] + self.assertEqual(rfc_entry.find(f"{ns}title").text, self.rfc.title) + self.assertEqual( + rfc_entry.find(f"{ns}current-status").text, "INTERNET STANDARD" + ) + self.assertEqual( + rfc_entry.find(f"{ns}publication-status").text, "PROPOSED STANDARD" + ) + self.assertEqual( + [(c.tag, c.text) for c in rfc_entry.find(f"{ns}date")], + [(f"{ns}month", "April"), (f"{ns}year", "2021")], + ) + + +class HelperTests(TestCase): + def test_format_rfc_number(self): + self.assertEqual(format_rfc_number(10), "10") + with override_settings(RFCINDEX_MATCH_LEGACY_XML=True): + self.assertEqual(format_rfc_number(10), "0010") + + def test_save_to_red_bucket(self): + red_bucket = storages["red_bucket"] + with override_settings(RFCINDEX_DELETE_THEN_WRITE=False): + save_to_red_bucket("test", StringIO("contents")) + with red_bucket.open("test", "r") as f: + self.assertEqual(f.read(), "contents") + with override_settings(RFCINDEX_DELETE_THEN_WRITE=True): + save_to_red_bucket("test", BytesIO(b"new contents")) + with red_bucket.open("test", "r") as f: + self.assertEqual(f.read(), "new contents") + red_bucket.delete("test") # clean up like a good child + + def test_get_unusable_rfc_numbers_raises(self): + """get_unusable_rfc_numbers should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_unusable_rfc_numbers() + red_bucket = storages["red_bucket"] + red_bucket.save("unusable-rfc-numbers.json", StringIO("not json")) + with self.assertRaises(json.JSONDecodeError): + get_unusable_rfc_numbers() + red_bucket.delete("unusable-rfc-numbers.json") + + def test_get_april1_rfc_numbers_raises(self): + """get_april1_rfc_numbers should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_april1_rfc_numbers() + red_bucket = storages["red_bucket"] + red_bucket.save("april-first-rfc-numbers.json", StringIO("not json")) + with self.assertRaises(json.JSONDecodeError): + get_april1_rfc_numbers() + red_bucket.delete("april-first-rfc-numbers.json") + + def test_get_publication_std_levels_raises(self): + """get_publication_std_levels should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_publication_std_levels() + red_bucket = storages["red_bucket"] + red_bucket.save("publication-std-levels.json", StringIO("not json")) + with self.assertRaises(json.JSONDecodeError): + get_publication_std_levels() + red_bucket.delete("publication-std-levels.json") diff --git a/ietf/templates/sync/rfc-index.txt b/ietf/templates/sync/rfc-index.txt new file mode 100644 index 0000000000..0f01ddfa90 --- /dev/null +++ b/ietf/templates/sync/rfc-index.txt @@ -0,0 +1,69 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + RFC INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all RFCs in numeric order. + +RFC citations appear in this format: + + #### Title of RFC. Author 1, Author 2, Author 3. Issue date. + (Format: ASCII) (Obsoletes xxx) (Obsoleted by xxx) (Updates xxx) + (Updated by xxx) (Also FYI ####) (Status: ssssss) (DOI: ddd) + +or + + #### Not Issued. + +For example: + + 1129 Internet Time Synchronization: The Network Time Protocol. D.L. + Mills. October 1989. (Format: TXT, PS, PDF, HTML) (Also RFC1119) + (Status: INFORMATIONAL) (DOI: 10.17487/RFC1129) + +Key to citations: + +#### is the RFC number. + +Following the RFC number are the title, the author(s), and the +publication date of the RFC. Each of these is terminated by a period. + +Following the number are the title (terminated with a period), the +author, or list of authors (terminated with a period), and the date +(terminated with a period). + +The format follows in parentheses. One or more of the following formats +are listed: text (TXT), PostScript (PS), Portable Document Format +(PDF), HTML, XML. + +Obsoletes xxxx refers to other RFCs that this one replaces; +Obsoleted by xxxx refers to RFCs that have replaced this one. +Updates xxxx refers to other RFCs that this one merely updates (but +does not replace); Updated by xxxx refers to RFCs that have updated +(but not replaced) this one. Generally, only immediately succeeding +and/or preceding RFCs are indicated, not the entire history of each +related earlier or later RFC in a related series. + +The (Also FYI ##) or (Also STD ##) or (Also BCP ##) phrase gives the +equivalent FYI, STD, or BCP number if the RFC is also in those +document sub-series. The Status field gives the document's +current status (see RFC 2026). The (DOI ddd) field gives the +Digital Object Identifier. + +RFCs may be obtained in a number of ways, using HTTP, FTP, or email. +See the RFC Editor Web page http://www.rfc-editor.org + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + RFC INDEX + --------- + + + +{% for rfc in rfcs %}{{rfc|safe}} + +{% endfor %} diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 0386dbbdf9..5ca4ba5cd9 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -417,6 +417,39 @@ def _multiline_to_list(s): ), } +# Configure storage for the red bucket - assume it uses the same credentials as +# other blobs +_red_bucket_name = os.environ.get("DATATRACKER_BLOB_STORE_RED_BUCKET_NAME", "").strip() +if _red_bucket_name == "": + raise RuntimeError("DATATRACKER_BLOB_STORE_RED_BUCKET_NAME must be set") + +STORAGES["red_bucket"] = { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": dict( + endpoint_url=_blob_store_endpoint_url, + access_key=_blob_store_access_key, + secret_key=_blob_store_secret_key, + security_token=None, + client_config=botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + signature_version="s3v4", + connect_timeout=_blob_store_connect_timeout, + read_timeout=_blob_store_read_timeout, + retries={"total_max_attempts": _blob_store_max_attempts}, + ), + verify=False, + bucket_name=_red_bucket_name, + ), +} +RFCINDEX_DELETE_THEN_WRITE = False # S3Storage allows file_overwrite by default +RFCINDEX_OUTPUT_PATH = os.environ.get( + "DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/" +) +RFCINDEX_INPUT_PATH = os.environ.get( + "DATATRACKR_RFCINDEX_INPUT_PATH", "" +) + # Configure the blobdb app for artifact storage _blobdb_replication_enabled = ( os.environ.get("DATATRACKER_BLOBDB_REPLICATION_ENABLED", "true").lower() == "true" From c226749c301fbecfd5503ebd66ba3692187d2946 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 12 Mar 2026 16:31:56 -0300 Subject: [PATCH 086/158] feat: task + API for rfc-index creation (#10537) * chore: fix typo in k8s/settings_local.py * feat: refresh_rfc_index() API * fix: use ContentFile, manually encode str Works better with S3Storage * chore(dev): expose blobstore on fixed ports Simplifies connecting purple to the blob store * chore(dev): typo * test: fix + test encoding more carefully * test: cover the new url --- .devcontainer/docker-compose.extend.yml | 4 +-- docker/docker-compose.extend.yml | 4 +-- ietf/api/tests_views_rpc.py | 15 ++++++++++ ietf/api/urls_rpc.py | 5 ++++ ietf/api/views_rpc.py | 16 +++++++++++ ietf/sync/rfcindex.py | 17 ++++++----- ietf/sync/tasks.py | 10 ++++++- ietf/sync/tests_rfcindex.py | 38 ++++++++++++++----------- k8s/settings_local.py | 2 +- 9 files changed, 82 insertions(+), 29 deletions(-) diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml index a92f42bc6d..ce1ce259fd 100644 --- a/.devcontainer/docker-compose.extend.yml +++ b/.devcontainer/docker-compose.extend.yml @@ -14,8 +14,8 @@ services: network_mode: service:db blobstore: ports: - - '9000' - - '9001' + - '9000:9000' + - '9001:9001' volumes: datatracker-vscode-ext: diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml index a69a453110..12ebe447d5 100644 --- a/docker/docker-compose.extend.yml +++ b/docker/docker-compose.extend.yml @@ -18,8 +18,8 @@ services: - '5433' blobstore: ports: - - '9000' - - '9001' + - '9000:9000' + - '9001:9001' celery: volumes: - .:/workspace diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 6a5a5c9b88..7ab8778d28 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -363,3 +363,18 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 200) # conflict + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.create_rfc_index_task") + def test_refresh_rfc_index(self, mock_task): + url = urlreverse("ietf.api.purple_api.refresh_rfc_index") + response = self.client.get(url) + self.assertEqual(response.status_code, 403) + response = self.client.get(url, headers={"X-Api-Key": "invalid-token"}) + self.assertEqual(response.status_code, 403) + response = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(response.status_code, 405) + self.assertFalse(mock_task.delay.called) + response = self.client.post(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(response.status_code, 202) + self.assertTrue(mock_task.delay.called) diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index 9d41ac137f..8555610dc3 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -30,6 +30,11 @@ views_rpc.RfcPubFilesView.as_view(), name="ietf.api.purple_api.upload_rfc_files", ), + path( + r"rfc_index/refresh/", + views_rpc.RfcIndexView.as_view(), + name="ietf.api.purple_api.refresh_rfc_index", + ), path(r"subject//person/", views_rpc.SubjectPersonView.as_view()), ] diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 8862bbf866..c7ae699005 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -40,6 +40,7 @@ from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage from ietf.doc.tasks import signal_update_rfc_metadata_task from ietf.person.models import Email, Person +from ietf.sync.tasks import create_rfc_index_task class Conflict(APIException): @@ -516,3 +517,18 @@ def post(self, request): shutil.move(ftm, destination) return Response(NotificationAckSerializer().data) + + +class RfcIndexView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="refresh_rfc_index", + summary="Refresh rfc-index files", + description="Requests creation of rfc-index.xml and rfc-index.txt files", + responses={202: None}, + request=None, + ) + def post(self, request): + create_rfc_index_task.delay() + return Response(status=202) diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index b15846094f..63c2044931 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -3,7 +3,6 @@ from collections import defaultdict from collections.abc import Container from dataclasses import dataclass -from io import StringIO, BytesIO from itertools import chain from operator import attrgetter, itemgetter from pathlib import Path @@ -11,6 +10,7 @@ from urllib.parse import urljoin from django.conf import settings +from django.core.files.base import ContentFile from lxml import etree from django.core.files.storage import storages @@ -28,7 +28,7 @@ def format_rfc_number(n): """Format an RFC number (or subseries doc number) - + Set settings.RFCINDEX_MATCH_LEGACY_XML=True for the legacy (leading-zero) format. That is for debugging only - tests will fail. """ @@ -42,13 +42,16 @@ def errata_url(rfc: Document): return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}") -def save_to_red_bucket(filename: str, content: BytesIO | StringIO): +def save_to_red_bucket(filename: str, content: str | bytes): red_bucket = storages["red_bucket"] bucket_path = str(Path(getattr(settings, "RFCINDEX_OUTPUT_PATH", "")) / filename) if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True): # Django 4.2's FileSystemStorage does not support allow_overwrite. red_bucket.delete(bucket_path) - red_bucket.save(bucket_path, content) + red_bucket.save( + bucket_path, + ContentFile(content if isinstance(content, bytes) else content.encode("utf-8")), + ) log(f"Saved {bucket_path} in red_bucket storage") @@ -76,7 +79,7 @@ def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: except json.JSONDecodeError: log(f"Error: unable to parse {bucket_path} in red_bucket storage") if settings.SERVER_MODE == "development": - return [] # pragma: no cover + return [] # pragma: no cover raise assert all(isinstance(record["number"], int) for record in records) assert all(isinstance(record["comment"], str) for record in records) @@ -441,7 +444,7 @@ def create_rfc_txt_index(): "rfcs": get_rfc_text_index_entries(), }, ) - save_to_red_bucket("rfc-index.txt", StringIO(index)) + save_to_red_bucket("rfc-index.txt", index) def create_rfc_xml_index(): @@ -477,4 +480,4 @@ def create_rfc_xml_index(): xml_declaration=True, pretty_print=4, ) - save_to_red_bucket("rfc-index.xml", BytesIO(pretty_index)) + save_to_red_bucket("rfc-index.xml", pretty_index) diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index fc75a056ed..4c84dc581e 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # @@ -18,6 +18,7 @@ from ietf.sync import iana from ietf.sync import rfceditor from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue +from ietf.sync.rfcindex import create_rfc_txt_index, create_rfc_xml_index from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper from ietf.utils import log from ietf.utils.timezone import date_today @@ -272,3 +273,10 @@ def load_rfcs_into_blobdb_task(start: int, end: int): if end > 11000: # Arbitrarily chosen end = 11000 load_rfcs_into_blobdb(list(range(start, end + 1))) + + +@shared_task +def create_rfc_index_task(): + create_rfc_txt_index() + create_rfc_xml_index() + diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index b0a8712fe1..e682c016f5 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -1,8 +1,8 @@ # Copyright The IETF Trust 2026, All Rights Reserved import json -from io import BytesIO, StringIO from unittest import mock +from django.core.files.base import ContentFile from django.core.files.storage import storages from django.test.utils import override_settings from lxml import etree @@ -13,7 +13,9 @@ create_rfc_txt_index, create_rfc_xml_index, format_rfc_number, - save_to_red_bucket, get_unusable_rfc_numbers, get_april1_rfc_numbers, + save_to_red_bucket, + get_unusable_rfc_numbers, + get_april1_rfc_numbers, get_publication_std_levels, ) from ietf.utils.test_utils import TestCase @@ -39,7 +41,7 @@ def setUp(self): # Create an unused RFC number red_bucket.save( "input/unusable-rfc-numbers.json", - StringIO(json.dumps([{"number": 123, "comment": ""}])), + ContentFile(json.dumps([{"number": 123, "comment": ""}])), ) # actual April 1 RFC @@ -55,7 +57,7 @@ def setUp(self): # Set up a JSON file to flag the April 1 RFC red_bucket.save( "input/april-first-rfc-numbers.json", - StringIO(json.dumps([self.april_fools_rfc.rfc_number])), + ContentFile(json.dumps([self.april_fools_rfc.rfc_number])), ) # non-April Fools RFC that happens to have been published on April 1 @@ -71,7 +73,7 @@ def setUp(self): # standard of self.rfc as different from its current value red_bucket.save( "input/publication-std-levels.json", - StringIO( + ContentFile( json.dumps( [{"number": self.rfc.rfc_number, "publication_std_level": "ps"}] ) @@ -91,7 +93,8 @@ def test_create_rfc_txt_index(self, mock_save): create_rfc_txt_index() self.assertEqual(mock_save.call_count, 1) self.assertEqual(mock_save.call_args[0][0], "rfc-index.txt") - contents = mock_save.call_args[0][1].read() + contents = mock_save.call_args[0][1] + self.assertTrue(isinstance(contents, str)) self.assertIn( "123 Not Issued.", contents, @@ -119,7 +122,8 @@ def test_create_rfc_xml_index(self, mock_save): create_rfc_xml_index() self.assertEqual(mock_save.call_count, 1) self.assertEqual(mock_save.call_args[0][0], "rfc-index.xml") - contents = mock_save.call_args[0][1].read() + contents = mock_save.call_args[0][1] + self.assertTrue(isinstance(contents, bytes)) ns = "{https://www.rfc-editor.org/rfc-index}" # NOT an f-string index = etree.fromstring(contents) @@ -190,13 +194,15 @@ def test_format_rfc_number(self): def test_save_to_red_bucket(self): red_bucket = storages["red_bucket"] with override_settings(RFCINDEX_DELETE_THEN_WRITE=False): - save_to_red_bucket("test", StringIO("contents")) - with red_bucket.open("test", "r") as f: - self.assertEqual(f.read(), "contents") + save_to_red_bucket("test", "contents \U0001f600") + # Read as binary and explicitly decode to confirm encoding + with red_bucket.open("test", "rb") as f: + self.assertEqual(f.read().decode("utf-8"), "contents \U0001f600") with override_settings(RFCINDEX_DELETE_THEN_WRITE=True): - save_to_red_bucket("test", BytesIO(b"new contents")) - with red_bucket.open("test", "r") as f: - self.assertEqual(f.read(), "new contents") + save_to_red_bucket("test", "new contents \U0001fae0".encode("utf-8")) + # Read as binary and explicitly decode to confirm encoding + with red_bucket.open("test", "rb") as f: + self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0") red_bucket.delete("test") # clean up like a good child def test_get_unusable_rfc_numbers_raises(self): @@ -204,7 +210,7 @@ def test_get_unusable_rfc_numbers_raises(self): with self.assertRaises(FileNotFoundError): get_unusable_rfc_numbers() red_bucket = storages["red_bucket"] - red_bucket.save("unusable-rfc-numbers.json", StringIO("not json")) + red_bucket.save("unusable-rfc-numbers.json", ContentFile("not json")) with self.assertRaises(json.JSONDecodeError): get_unusable_rfc_numbers() red_bucket.delete("unusable-rfc-numbers.json") @@ -214,7 +220,7 @@ def test_get_april1_rfc_numbers_raises(self): with self.assertRaises(FileNotFoundError): get_april1_rfc_numbers() red_bucket = storages["red_bucket"] - red_bucket.save("april-first-rfc-numbers.json", StringIO("not json")) + red_bucket.save("april-first-rfc-numbers.json", ContentFile("not json")) with self.assertRaises(json.JSONDecodeError): get_april1_rfc_numbers() red_bucket.delete("april-first-rfc-numbers.json") @@ -224,7 +230,7 @@ def test_get_publication_std_levels_raises(self): with self.assertRaises(FileNotFoundError): get_publication_std_levels() red_bucket = storages["red_bucket"] - red_bucket.save("publication-std-levels.json", StringIO("not json")) + red_bucket.save("publication-std-levels.json", ContentFile("not json")) with self.assertRaises(json.JSONDecodeError): get_publication_std_levels() red_bucket.delete("publication-std-levels.json") diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 5ca4ba5cd9..56e395c5ac 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -447,7 +447,7 @@ def _multiline_to_list(s): "DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/" ) RFCINDEX_INPUT_PATH = os.environ.get( - "DATATRACKR_RFCINDEX_INPUT_PATH", "" + "DATATRACKER_RFCINDEX_INPUT_PATH", "" ) # Configure the blobdb app for artifact storage From 2c59afe783216285b9695e99ee64547fe4e66469 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 13 Mar 2026 16:14:38 -0300 Subject: [PATCH 087/158] fix: drop stale obs/updates in rfced sync (#10543) * fix: drop stale obs/updates in rfced sync * refactor: partial revert, orig was safer --- ietf/sync/rfceditor.py | 73 +++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py index cdcdeb5989..aa0e643b20 100644 --- a/ietf/sync/rfceditor.py +++ b/ietf/sync/rfceditor.py @@ -636,43 +636,70 @@ def update_docs_from_rfc_index( ) rfc_published = True - def parse_relation_list(l): - res = [] - for x in l: - for a in Document.objects.filter(name=x.lower(), type_id="rfc"): - if a not in res: - res.append(a) - return res - - for x in parse_relation_list(obsoletes): - if not RelatedDocument.objects.filter( - source=doc, target=x, relationship=relationship_obsoletes + def parse_relation_list(rel_list: list[str]) -> list[Document]: + return list( + Document.objects.filter( + name__in=[name.strip().lower() for name in rel_list], + type_id="rfc" + ) + ) + + # Create missing obsoletes relations + docs_this_obsoletes = parse_relation_list(obsoletes) + for obs_doc in docs_this_obsoletes: + if not doc.relateddocument_set.filter( + target=obs_doc, relationship=relationship_obsoletes ): - r = RelatedDocument.objects.create( - source=doc, target=x, relationship=relationship_obsoletes + r = doc.relateddocument_set.create( + target=obs_doc, relationship=relationship_obsoletes ) rfc_changes.append( - "created {rel_name} relation between {src_name} and {tgt_name}".format( + "created {rel_name} relation between {src} and {tgt}".format( rel_name=r.relationship.name.lower(), - src_name=prettify_std_name(r.source.name), - tgt_name=prettify_std_name(r.target.name), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), ) ) + # Remove stale obsoletes relations + for r in doc.relateddocument_set.filter( + relationship=relationship_obsoletes + ).exclude(target_id__in=[d.pk for d in docs_this_obsoletes]): + r.delete() + rfc_changes.append( + "removed {rel_name} relation between {src} and {tgt}".format( + rel_name=r.relationship.name.lower(), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), + ) + ) - for x in parse_relation_list(updates): + docs_this_updates = parse_relation_list(updates) + for upd_doc in docs_this_updates: if not RelatedDocument.objects.filter( - source=doc, target=x, relationship=relationship_updates + source=doc, target=upd_doc, relationship=relationship_updates ): - r = RelatedDocument.objects.create( - source=doc, target=x, relationship=relationship_updates + r = doc.relateddocument_set.create( + target=upd_doc, relationship=relationship_updates ) rfc_changes.append( - "created {rel_name} relation between {src_name} and {tgt_name}".format( + "created {rel_name} relation between {src} and {tgt}".format( rel_name=r.relationship.name.lower(), - src_name=prettify_std_name(r.source.name), - tgt_name=prettify_std_name(r.target.name), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), ) ) + # Remove stale updates relations + for r in doc.relateddocument_set.filter( + relationship=relationship_updates + ).exclude(target_id__in=[d.pk for d in docs_this_updates]): + r.delete() + rfc_changes.append( + "removed {rel_name} relation between {src} and {tgt}".format( + rel_name=r.relationship.name.lower(), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), + ) + ) if also: # recondition also to have proper subseries document names: From 76fd25a1f39093a214be8ac2e0a9ed452beb7a47 Mon Sep 17 00:00:00 2001 From: Tianyi Gao Date: Sat, 14 Mar 2026 12:19:51 +0800 Subject: [PATCH 088/158] fix: wording in id_expired_email template (#10154) --- ietf/templates/doc/draft/id_expired_email.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/templates/doc/draft/id_expired_email.txt b/ietf/templates/doc/draft/id_expired_email.txt index afbf253ee2..161146a301 100644 --- a/ietf/templates/doc/draft/id_expired_email.txt +++ b/ietf/templates/doc/draft/id_expired_email.txt @@ -1,4 +1,4 @@ -{% autoescape off %}{{ doc.file_tag|safe }} was just expired. +{% autoescape off %}{{ doc.file_tag|safe }} just expired. This Internet-Draft is in the state "{{ state }}" in the Datatracker. From 9646edc20378e101ab48ff24253861fc5ea78fe9 Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Sun, 15 Mar 2026 02:17:40 +0800 Subject: [PATCH 089/158] feat: add author affiliation in serializer (#10549) --- ietf/api/serializers_rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index c17cbc64ce..a18dc588c4 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -103,7 +103,7 @@ class DocumentAuthorSerializer(serializers.ModelSerializer): class Meta: model = DocumentAuthor - fields = ["person", "plain_name"] + fields = ["person", "plain_name", "affiliation"] def get_plain_name(self, document_author: DocumentAuthor) -> str: return document_author.person.plain_name() From 36fa518ec387f425b1f11f6f9040a73e8f61df30 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:02:47 -0400 Subject: [PATCH 090/158] chore(deps): bump the npm group across /dev/deploy-to-container with 5 updates (#10560) Bumps the npm group with 5 updates in the /dev/deploy-to-container directory: | Package | From | To | | --- | --- | --- | | [dockerode](https://github.com/apocas/dockerode) | `4.0.6` | `4.0.9` | | [fs-extra](https://github.com/jprichardson/node-fs-extra) | `11.3.0` | `11.3.4` | | [nanoid](https://github.com/ai/nanoid) | `5.1.5` | `5.1.7` | | [slugify](https://github.com/simov/slugify) | `1.6.6` | `1.6.8` | | [tar](https://github.com/isaacs/node-tar) | `7.4.3` | `7.5.11` | Updates `dockerode` from 4.0.6 to 4.0.9 - [Release notes](https://github.com/apocas/dockerode/releases) - [Commits](https://github.com/apocas/dockerode/compare/v4.0.6...v4.0.9) Updates `fs-extra` from 11.3.0 to 11.3.4 - [Changelog](https://github.com/jprichardson/node-fs-extra/blob/master/CHANGELOG.md) - [Commits](https://github.com/jprichardson/node-fs-extra/compare/11.3.0...11.3.4) Updates `nanoid` from 5.1.5 to 5.1.7 - [Release notes](https://github.com/ai/nanoid/releases) - [Changelog](https://github.com/ai/nanoid/blob/main/CHANGELOG.md) - [Commits](https://github.com/ai/nanoid/compare/5.1.5...5.1.7) Updates `slugify` from 1.6.6 to 1.6.8 - [Changelog](https://github.com/simov/slugify/blob/master/CHANGELOG.md) - [Commits](https://github.com/simov/slugify/compare/v1.6.6...v1.6.8) Updates `tar` from 7.4.3 to 7.5.11 - [Release notes](https://github.com/isaacs/node-tar/releases) - [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md) - [Commits](https://github.com/isaacs/node-tar/compare/v7.4.3...v7.5.11) --- updated-dependencies: - dependency-name: dockerode dependency-version: 4.0.9 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: npm - dependency-name: fs-extra dependency-version: 11.3.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: npm - dependency-name: nanoid dependency-version: 5.1.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: npm - dependency-name: slugify dependency-version: 1.6.8 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: npm - dependency-name: tar dependency-version: 7.5.11 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: npm ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dev/deploy-to-container/package-lock.json | 733 +++------------------- dev/deploy-to-container/package.json | 10 +- 2 files changed, 76 insertions(+), 667 deletions(-) diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json index 0954ec9af4..b62109f0e2 100644 --- a/dev/deploy-to-container/package-lock.json +++ b/dev/deploy-to-container/package-lock.json @@ -6,12 +6,12 @@ "": { "name": "deploy-to-container", "dependencies": { - "dockerode": "^4.0.6", - "fs-extra": "^11.3.0", - "nanoid": "5.1.5", + "dockerode": "^4.0.9", + "fs-extra": "^11.3.4", + "nanoid": "5.1.7", "nanoid-dictionary": "5.0.0", - "slugify": "1.6.6", - "tar": "^7.4.3", + "slugify": "1.6.8", + "tar": "^7.5.11", "yargs": "^17.7.2" }, "engines": { @@ -52,95 +52,6 @@ "node": ">=6" } }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" - }, - "node_modules/@isaacs/cliui/node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@isaacs/cliui/node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, "node_modules/@isaacs/fs-minipass": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", @@ -161,15 +72,6 @@ "url": "https://opencollective.com/js-sdsl" } }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "optional": true, - "engines": { - "node": ">=14" - } - }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -263,11 +165,6 @@ "safer-buffer": "~2.1.0" } }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -285,8 +182,7 @@ "type": "consulting", "url": "https://feross.org/support" } - ], - "license": "MIT" + ] }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", @@ -301,21 +197,12 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "license": "MIT", "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", "readable-stream": "^3.4.0" } }, - "node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -334,7 +221,6 @@ "url": "https://feross.org/support" } ], - "license": "MIT", "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -352,8 +238,7 @@ "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "license": "ISC" + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, "node_modules/cliui": { "version": "8.0.1", @@ -398,19 +283,6 @@ "node": ">=10.0.0" } }, - "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/debug": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", @@ -444,38 +316,31 @@ } }, "node_modules/dockerode": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz", - "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==", - "license": "Apache-2.0", + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz", + "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==", "dependencies": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", "docker-modem": "^5.0.6", "protobufjs": "^7.3.2", - "tar-fs": "~2.1.2", + "tar-fs": "^2.1.4", "uuid": "^10.0.0" }, "engines": { "node": ">= 8.0" } }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" - }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "license": "MIT", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", "dependencies": { "once": "^1.4.0" } @@ -488,32 +353,15 @@ "node": ">=6" } }, - "node_modules/foreground-child": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", - "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", - "dependencies": { - "cross-spawn": "^7.0.0", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "license": "MIT" + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "node_modules/fs-extra": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz", - "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==", - "license": "MIT", + "version": "11.3.4", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.4.tgz", + "integrity": "sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==", "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -531,27 +379,6 @@ "node": "6.* || 8.* || >= 10.*" } }, - "node_modules/glob": { - "version": "10.3.12", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", - "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^2.3.6", - "minimatch": "^9.0.1", - "minipass": "^7.0.4", - "path-scurry": "^1.10.2" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -574,8 +401,7 @@ "type": "consulting", "url": "https://feross.org/support" } - ], - "license": "BSD-3-Clause" + ] }, "node_modules/inherits": { "version": "2.0.4", @@ -590,28 +416,6 @@ "node": ">=8" } }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" - }, - "node_modules/jackspeak": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", - "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" - } - }, "node_modules/jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -633,28 +437,6 @@ "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz", "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg==" }, - "node_modules/lru-cache": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", - "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", - "engines": { - "node": "14 || >=16.14" - } - }, - "node_modules/minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", @@ -664,36 +446,20 @@ } }, "node_modules/minizlib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", - "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", "dependencies": { - "minipass": "^7.0.4", - "rimraf": "^5.0.5" + "minipass": "^7.1.2" }, "engines": { "node": ">= 18" } }, - "node_modules/mkdirp": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", - "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", - "bin": { - "mkdirp": "dist/cjs/src/bin.js" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "license": "MIT" + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, "node_modules/ms": { "version": "2.1.3", @@ -709,16 +475,15 @@ "optional": true }, "node_modules/nanoid": { - "version": "5.1.5", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz", - "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw==", + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.7.tgz", + "integrity": "sha512-ua3NDgISf6jdwezAheMOk4mbE1LXjm1DfMUDMuJf4AqxLFK3ccGpgWizwa5YV7Yz9EpXwEaWoRXSb/BnV0t5dQ==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/ai" } ], - "license": "MIT", "bin": { "nanoid": "bin/nanoid.js" }, @@ -736,34 +501,10 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "license": "ISC", "dependencies": { "wrappy": "1" } }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-scurry": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", - "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", - "dependencies": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/protobufjs": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", @@ -788,10 +529,9 @@ } }, "node_modules/pump": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", - "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", - "license": "MIT", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -818,23 +558,6 @@ "node": ">=0.10.0" } }, - "node_modules/rimraf": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", - "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", - "dependencies": { - "glob": "^10.3.7" - }, - "bin": { - "rimraf": "dist/esm/bin.mjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -860,40 +583,10 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "engines": { - "node": ">=8" - } - }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/slugify": { - "version": "1.6.6", - "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", - "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==", + "version": "1.6.8", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.8.tgz", + "integrity": "sha512-HVk9X1E0gz3mSpoi60h/saazLKXKaZThMLU3u/aNwoYn8/xQyX2MGxL0ui2eaokkD7tF+Zo+cKTHUbe1mmmGzA==", "engines": { "node": ">=8.0.0" } @@ -942,20 +635,6 @@ "node": ">=8" } }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -967,28 +646,15 @@ "node": ">=8" } }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/tar": { - "version": "7.4.3", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", - "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "version": "7.5.11", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz", + "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", - "minizlib": "^3.0.1", - "mkdirp": "^3.0.1", + "minizlib": "^3.1.0", "yallist": "^5.0.0" }, "engines": { @@ -996,10 +662,9 @@ } }, "node_modules/tar-fs": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz", - "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==", - "license": "MIT", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -1011,7 +676,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "license": "MIT", "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -1067,20 +731,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -1097,28 +747,10 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "license": "ISC" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/y18n": { "version": "5.0.8", @@ -1188,64 +820,6 @@ "yargs": "^17.7.2" } }, - "@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "requires": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" - }, - "ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==" - }, - "emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" - }, - "string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "requires": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - } - }, - "strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "requires": { - "ansi-regex": "^6.0.1" - } - }, - "wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "requires": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - } - } - } - }, "@isaacs/fs-minipass": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", @@ -1259,12 +833,6 @@ "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==" }, - "@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "optional": true - }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -1348,11 +916,6 @@ "safer-buffer": "~2.1.0" } }, - "balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1376,14 +939,6 @@ "readable-stream": "^3.4.0" } }, - "brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "requires": { - "balanced-match": "^1.0.0" - } - }, "buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -1437,16 +992,6 @@ "nan": "^2.19.0" } }, - "cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "requires": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - } - }, "debug": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", @@ -1467,33 +1012,28 @@ } }, "dockerode": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz", - "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==", + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz", + "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==", "requires": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", "docker-modem": "^5.0.6", "protobufjs": "^7.3.2", - "tar-fs": "~2.1.2", + "tar-fs": "^2.1.4", "uuid": "^10.0.0" } }, - "eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" - }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", "requires": { "once": "^1.4.0" } @@ -1503,24 +1043,15 @@ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, - "foreground-child": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", - "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", - "requires": { - "cross-spawn": "^7.0.0", - "signal-exit": "^4.0.1" - } - }, "fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "fs-extra": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz", - "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==", + "version": "11.3.4", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.4.tgz", + "integrity": "sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==", "requires": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -1532,18 +1063,6 @@ "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, - "glob": { - "version": "10.3.12", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", - "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", - "requires": { - "foreground-child": "^3.1.0", - "jackspeak": "^2.3.6", - "minimatch": "^9.0.1", - "minipass": "^7.0.4", - "path-scurry": "^1.10.2" - } - }, "graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -1564,20 +1083,6 @@ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, - "isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" - }, - "jackspeak": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", - "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", - "requires": { - "@isaacs/cliui": "^8.0.2", - "@pkgjs/parseargs": "^0.11.0" - } - }, "jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -1597,38 +1102,19 @@ "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz", "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg==" }, - "lru-cache": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", - "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==" - }, - "minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "requires": { - "brace-expansion": "^2.0.1" - } - }, "minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==" }, "minizlib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", - "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", "requires": { - "minipass": "^7.0.4", - "rimraf": "^5.0.5" + "minipass": "^7.1.2" } }, - "mkdirp": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", - "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==" - }, "mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", @@ -1646,9 +1132,9 @@ "optional": true }, "nanoid": { - "version": "5.1.5", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz", - "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw==" + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.7.tgz", + "integrity": "sha512-ua3NDgISf6jdwezAheMOk4mbE1LXjm1DfMUDMuJf4AqxLFK3ccGpgWizwa5YV7Yz9EpXwEaWoRXSb/BnV0t5dQ==" }, "nanoid-dictionary": { "version": "5.0.0", @@ -1663,20 +1149,6 @@ "wrappy": "1" } }, - "path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" - }, - "path-scurry": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", - "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", - "requires": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - } - }, "protobufjs": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", @@ -1697,9 +1169,9 @@ } }, "pump": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", - "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -1720,14 +1192,6 @@ "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" }, - "rimraf": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", - "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", - "requires": { - "glob": "^10.3.7" - } - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1738,28 +1202,10 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "requires": { - "shebang-regex": "^3.0.0" - } - }, - "shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" - }, - "signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" - }, "slugify": { - "version": "1.6.6", - "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", - "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==" + "version": "1.6.8", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.8.tgz", + "integrity": "sha512-HVk9X1E0gz3mSpoi60h/saazLKXKaZThMLU3u/aNwoYn8/xQyX2MGxL0ui2eaokkD7tF+Zo+cKTHUbe1mmmGzA==" }, "split-ca": { "version": "1.0.1", @@ -1795,16 +1241,6 @@ "strip-ansi": "^6.0.1" } }, - "string-width-cjs": { - "version": "npm:string-width@4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -1813,24 +1249,15 @@ "ansi-regex": "^5.0.1" } }, - "strip-ansi-cjs": { - "version": "npm:strip-ansi@6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - }, "tar": { - "version": "7.4.3", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", - "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "version": "7.5.11", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz", + "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==", "requires": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", - "minizlib": "^3.0.1", - "mkdirp": "^3.0.1", + "minizlib": "^3.1.0", "yallist": "^5.0.0" }, "dependencies": { @@ -1842,9 +1269,9 @@ } }, "tar-fs": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz", - "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", "requires": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -1889,14 +1316,6 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==" }, - "which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "requires": { - "isexe": "^2.0.0" - } - }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -1907,16 +1326,6 @@ "strip-ansi": "^6.0.0" } }, - "wrap-ansi-cjs": { - "version": "npm:wrap-ansi@7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json index 09716c3094..1c95a4540c 100644 --- a/dev/deploy-to-container/package.json +++ b/dev/deploy-to-container/package.json @@ -2,12 +2,12 @@ "name": "deploy-to-container", "type": "module", "dependencies": { - "dockerode": "^4.0.6", - "fs-extra": "^11.3.0", - "nanoid": "5.1.5", + "dockerode": "^4.0.9", + "fs-extra": "^11.3.4", + "nanoid": "5.1.7", "nanoid-dictionary": "5.0.0", - "slugify": "1.6.6", - "tar": "^7.4.3", + "slugify": "1.6.8", + "tar": "^7.5.11", "yargs": "^17.7.2" }, "engines": { From dcce2df0300879078690a2fbd3522602d467cf38 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Fri, 20 Mar 2026 12:03:18 +0900 Subject: [PATCH 091/158] feat: add attendance summary and pie chart to meeting attendees page (#10481) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add attendance summary and pie chart to meeting attendees page For IETF meetings ≥ 118, the attendees proceedings page now shows an Onsite / Remote / Total summary row matching the counts displayed on registration.ietf.org, together with a "View chart" button that opens a Bootstrap modal containing a Highcharts pie chart. * Split out attendees-chart.js --- ietf/meeting/tests_views.py | 13 +++++ ietf/meeting/views.py | 33 +++++++++-- ietf/static/js/attendees-chart.js | 58 +++++++++++++++++++ .../meeting/proceedings_attendees.html | 53 ++++++++++++++++- package.json | 1 + 5 files changed, 151 insertions(+), 7 deletions(-) create mode 100644 ietf/static/js/attendees-chart.js diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 168999d0aa..258ffe554c 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -9007,6 +9007,8 @@ def test_proceedings_attendees(self): - assert onsite checkedin=True appears, not onsite checkedin=False - assert remote attended appears, not remote not attended - prefer onsite checkedin=True to remote attended when same person has both + - summary stats row shows correct counts + - chart data JSON is embedded with correct values """ m = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number="118") @@ -9028,6 +9030,17 @@ def test_proceedings_attendees(self): text = q('#id_attendees tbody tr').text().replace('\n', ' ') self.assertEqual(text, f"A Person {areg.affiliation} {areg.country_code} onsite C Person {creg.affiliation} {creg.country_code} remote") + # Summary stats row: Onsite / Remote / Total (matches registration.ietf.org) + self.assertContains(response, 'Onsite:') + self.assertContains(response, 'Remote:') + self.assertContains(response, 'Total:') + self.assertContains(response, '1') # onsite and remote + self.assertContains(response, '2') # total + + # Chart data embedded in page + chart_json = json.loads(q('#attendees-chart-data').text()) + self.assertEqual(chart_json['type'], [['Onsite', 1], ['Remote', 1]]) + def test_proceedings_overview(self): '''Test proceedings IETF Overview page. Note: old meetings aren't supported so need to add a new meeting then test. diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 731dfad88f..67a81305b4 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -109,7 +109,7 @@ from ietf.meeting.utils import get_activity_stats, post_process, create_recording, delete_recording from ietf.meeting.utils import participants_for_meeting, generate_bluesheet, bluesheet_data, save_bluesheet from ietf.message.utils import infer_message -from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName +from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName, CountryName from ietf.utils import markdown from ietf.utils.decorators import require_api_key from ietf.utils.hedgedoc import Note, NoteError @@ -4812,15 +4812,36 @@ def proceedings_attendees(request, num=None): template = None registrations = None + stats = None + chart_data = None + if int(meeting.number) >= 118: checked_in, attended = participants_for_meeting(meeting) regs = list(Registration.objects.onsite().filter(meeting__number=num, checkedin=True)) - - for reg in Registration.objects.remote().filter(meeting__number=num).select_related('person'): - if reg.person.pk in attended and reg.person.pk not in checked_in: - regs.append(reg) + onsite_count = len(regs) + regs += [ + reg + for reg in Registration.objects.remote().filter(meeting__number=num).select_related('person') + if reg.person.pk in attended and reg.person.pk not in checked_in + ] + remote_count = len(regs) - onsite_count registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name)) + + country_codes = [r.country_code for r in registrations if r.country_code] + stats = { + 'total': onsite_count + remote_count, + 'onsite': onsite_count, + 'remote': remote_count, + } + + code_to_name = dict(CountryName.objects.values_list('slug', 'name')) + country_counts = Counter(code_to_name.get(c, c) for c in country_codes).most_common() + + chart_data = { + 'type': [['Onsite', onsite_count], ['Remote', remote_count]], + 'countries': country_counts, + } else: overview_template = "/meeting/proceedings/%s/attendees.html" % meeting.number try: @@ -4832,6 +4853,8 @@ def proceedings_attendees(request, num=None): 'meeting': meeting, 'registrations': registrations, 'template': template, + 'stats': stats, + 'chart_data': chart_data, }) def proceedings_overview(request, num=None): diff --git a/ietf/static/js/attendees-chart.js b/ietf/static/js/attendees-chart.js new file mode 100644 index 0000000000..fed3b1289c --- /dev/null +++ b/ietf/static/js/attendees-chart.js @@ -0,0 +1,58 @@ +(function () { + var raw = document.getElementById('attendees-chart-data'); + if (!raw) return; + var chartData = JSON.parse(raw.textContent); + var chart = null; + var currentBreakdown = 'type'; + + // Override the global transparent background set by highcharts.js so the + // export menu and fullscreen view use the page background color. + var container = document.getElementById('attendees-pie-chart'); + var bodyBg = getComputedStyle(document.body).backgroundColor; + container.style.setProperty('--highcharts-background-color', bodyBg); + + function renderChart(breakdown) { + var seriesData = chartData[breakdown].map(function (item) { + return { name: item[0], y: item[1] }; + }); + if (chart) chart.destroy(); + chart = Highcharts.chart(container, { + chart: { type: 'pie', height: 400 }, + title: { text: null }, + tooltip: { pointFormat: '{point.name}: {point.y} ({point.percentage:.1f}%)' }, + plotOptions: { + pie: { + dataLabels: { + enabled: true, + format: '{point.name}
        {point.y} ({point.percentage:.1f}%)', + }, + showInLegend: false, + } + }, + series: [{ name: 'Attendees', data: seriesData }], + }); + } + + var modal = document.getElementById('attendees-chart-modal'); + + // Render (or re-render) the chart each time the modal becomes fully visible, + // so Highcharts can measure the container dimensions correctly. + modal.addEventListener('shown.bs.modal', function () { + renderChart(currentBreakdown); + }); + + // Release the chart when the modal closes to avoid stale renders. + modal.addEventListener('hidden.bs.modal', function () { + if (chart) { + chart.destroy(); + chart = null; + } + }); + + document.querySelectorAll('[name="attendees-breakdown"]').forEach(function (radio) { + radio.addEventListener('change', function () { + currentBreakdown = this.value; + renderChart(currentBreakdown); + }); + }); +})(); diff --git a/ietf/templates/meeting/proceedings_attendees.html b/ietf/templates/meeting/proceedings_attendees.html index 390ce00cad..0c59d4ab15 100644 --- a/ietf/templates/meeting/proceedings_attendees.html +++ b/ietf/templates/meeting/proceedings_attendees.html @@ -3,6 +3,7 @@ {% load origin markup_tags static %} {% block pagehead %} + {% if chart_data %}{% endif %} {% endblock %} {% block title %}IETF {{ meeting.number }} proceedings{% endblock %} {% block content %} @@ -14,8 +15,52 @@

        Attendee list of IETF {{ meeting.number }} meeting

        - + + {% if chart_data %} +
        +
        +
        Onsite: {{ stats.onsite }}
        +
        Remote: {{ stats.remote }}
        +
        Total: {{ stats.total }}
        +
        + +
        + + + + {{ chart_data|json_script:"attendees-chart-data" }} + {% endif %}{# chart_data #} + {% if template %} + {{template|safe}} {% else %}

        + {% if group.uses_milestone_dates %} Date {% else %} @@ -39,7 +39,7 @@

        {{ milestone.resolved|title }} {% else %} {% if group.uses_milestone_dates %} - {{ milestone.due|date:"M Y" }} + {{ milestone.due|date:"M Y" }} {% else %} {% if forloop.first %}Last{% endif %} {% if forloop.last %}Next{% endif %} From c8cf9e1e32c087b03b1fc9ce169a3c6f213c4e6a Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Fri, 7 Nov 2025 14:45:43 -0500 Subject: [PATCH 008/158] refactor: use ical for important-dates (#9849) * refactor: use ical for important-dates * chore: format and cleanup * fix: remove unwanted newline --- ietf/meeting/views.py | 74 +++++++++++++++++-- ietf/templates/meeting/important_dates.ics | 5 -- .../meeting/important_dates_for_meeting.ics | 24 ------ 3 files changed, 69 insertions(+), 34 deletions(-) delete mode 100644 ietf/templates/meeting/important_dates.ics delete mode 100644 ietf/templates/meeting/important_dates_for_meeting.ics diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 3484a85784..a4fe498152 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -4445,6 +4445,73 @@ def upcoming_ical(request): response['Content-Disposition'] = 'attachment; filename="upcoming.ics"' return response +def render_important_dates_ical(meetings, request): + """Generate important dates using the icalendar library""" + cal = Calendar() + cal.add("prodid", "-//IETF//datatracker.ietf.org ical importantdates//EN") + cal.add("version", "2.0") + cal.add("method", "PUBLISH") + + for meeting in meetings: + for important_date in meeting.important_dates: + event = Event() + event.add("uid", f"ietf-{meeting.number}-{important_date.name_id}-" + f"{important_date.date.isoformat()}") + event.add("summary", f"IETF {meeting.number}: {important_date.name.name}") + event.add("class", "PUBLIC") + + if not important_date.midnight_cutoff: + event.add("dtstart", important_date.date) + else: + event.add("dtstart", datetime.datetime.combine( + important_date.date, + datetime.time(23, 59, 0, tzinfo=pytz.UTC)) + ) + + event.add("transp", "TRANSPARENT") + event.add("dtstamp", meeting.cached_updated) + description_lines = [important_date.name.desc] + if important_date.name.slug in ('openreg', 'earlybird'): + description_lines.append( + "Register here: https://www.ietf.org/how/meetings/register/") + if important_date.name.slug == 'opensched': + description_lines.append("To request a Working Group session, use the " + "IETF Meeting Session Request Tool:") + description_lines.append(f"{request.scheme}://{request.get_host()}" + f"{reverse('ietf.meeting.views_session_request.list_view')}") + description_lines.append("If you are working on a BOF request, it is " + "highly recommended to tell the IESG") + description_lines.append("now by sending an email to iesg@ietf.org " + "to get advance help with the request.") + if important_date.name.slug == 'cutoffwgreq': + description_lines.append("To request a Working Group session, use the " + "IETF Meeting Session Request Tool:") + description_lines.append(f"{request.scheme}://{request.get_host()}" + f"{reverse('ietf.meeting.views_session_request.list_view')}") + if important_date.name.slug == 'cutoffbofreq': + description_lines.append("To request a BOF, please see instructions on " + "Requesting a BOF:") + description_lines.append("https://www.ietf.org/how/bofs/bof-procedures/") + if important_date.name.slug == 'idcutoff': + description_lines.append("Upload using the I-D Submission Tool:") + description_lines.append(f"{request.scheme}://{request.get_host()}" + f"{reverse('ietf.submit.views.upload_submission')}") + if important_date.name.slug in ( + 'draftwgagenda', + 'revwgagenda', + 'procsub', + 'revslug' + ): + description_lines.append("Upload using the Meeting Materials " + "Management Tool:") + description_lines.append(f"{request.scheme}://{request.get_host()}" + f"{reverse('ietf.meeting.views.materials', + kwargs={'num': meeting.number})}") + + event.add("description", "\n".join(description_lines)) + cal.add_component(event) + + return cal.to_ical().decode("utf-8") def upcoming_json(request): '''Return Upcoming meetings in json format''' @@ -5063,11 +5130,8 @@ def important_dates(request, num=None, output_format=None): if output_format == 'ics': preprocess_meeting_important_dates(meetings) - ics = render_to_string('meeting/important_dates.ics', { - 'meetings': meetings, - }, request=request) - # icalendar response file should have '\r\n' line endings per RFC5545 - response = HttpResponse(parse_ical_line_endings(ics), content_type='text/calendar') + response = HttpResponse(render_important_dates_ical(meetings, request), + content_type='text/calendar') response['Content-Disposition'] = 'attachment; filename="important-dates.ics"' return response diff --git a/ietf/templates/meeting/important_dates.ics b/ietf/templates/meeting/important_dates.ics deleted file mode 100644 index 35079e01eb..0000000000 --- a/ietf/templates/meeting/important_dates.ics +++ /dev/null @@ -1,5 +0,0 @@ -{% load humanize %}{% autoescape off %}{% load ietf_filters %}BEGIN:VCALENDAR -VERSION:2.0 -METHOD:PUBLISH -PRODID:-//IETF//datatracker.ietf.org ical importantdates//EN -{% for meeting in meetings %}{% include "meeting/important_dates_for_meeting.ics" %}{% endfor %}END:VCALENDAR{% endautoescape %} diff --git a/ietf/templates/meeting/important_dates_for_meeting.ics b/ietf/templates/meeting/important_dates_for_meeting.ics deleted file mode 100644 index e6d403da93..0000000000 --- a/ietf/templates/meeting/important_dates_for_meeting.ics +++ /dev/null @@ -1,24 +0,0 @@ -{# Copyright The IETF Trust 2025, All Rights Reserved #} -{% load tz ietf_filters %}{% for d in meeting.important_dates %}BEGIN:VEVENT -UID:ietf-{{ meeting.number }}-{{ d.name_id }}-{{ d.date.isoformat }} -SUMMARY:IETF {{ meeting.number }}: {{ d.name.name }} -CLASS:PUBLIC -DTSTART{% if not d.midnight_cutoff %};VALUE=DATE{% endif %}:{{ d.date|date:"Ymd" }}{% if d.midnight_cutoff %}235900Z{% endif %} -DTSTAMP{% ics_date_time meeting.cached_updated|utc 'utc' %} -TRANSP:TRANSPARENT -DESCRIPTION:{{ d.name.desc }}{% if first and d.name.slug == 'openreg' or first and d.name.slug == 'earlybird' %}\n - Register here: https://www.ietf.org/how/meetings/register/{% endif %}{% if d.name.slug == 'opensched' %}\n - To request a Working Group session, use the IETF Meeting Session Request Tool:\n - {{ request.scheme }}://{{ request.get_host}}{% url 'ietf.meeting.views_session_request.list_view' %}\n - If you are working on a BOF request, it is highly recommended to tell the IESG\n - now by sending an email to iesg@ietf.org to get advance help with the request.{% endif %}{% if d.name.slug == 'cutoffwgreq' %}\n - To request a Working Group session, use the IETF Meeting Session Request Tool:\n - {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.meeting.views_session_request.list_view' %}{% endif %}{% if d.name.slug == 'cutoffbofreq' %}\n - To request a BOF, please see instructions on Requesting a BOF:\n - https://www.ietf.org/how/bofs/bof-procedures/{% endif %}{% if d.name.slug == 'idcutoff' %}\n - Upload using the I-D Submission Tool:\n - {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.submit.views.upload_submission' %}{% endif %}{% if d.name.slug == 'draftwgagenda' or d.name.slug == 'revwgagenda' or d.name.slug == 'procsub' or d.name.slug == 'revslug' %}\n - Upload using the Meeting Materials Management Tool:\n - {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.meeting.views.materials' num=meeting.number %}{% endif %} -END:VEVENT -{% endfor %} From 2ea08c696a7317e74e24f50c8b85ffbf91cdfa7b Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Fri, 7 Nov 2025 14:46:46 -0500 Subject: [PATCH 009/158] Sort ADs by full name (#9839) --- ietf/doc/views_search.py | 3 ++- ietf/utils/unicodenormalize.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 ietf/utils/unicodenormalize.py diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 2144c23e06..3b67061b05 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -74,7 +74,7 @@ from ietf.utils.log import log from ietf.doc.utils_search import prepare_document_table, doc_type, doc_state, doc_type_name, AD_WORKLOAD from ietf.ietfauth.utils import has_role - +from ietf.utils.unicodenormalize import normalize_for_sorting class SearchForm(forms.Form): name = forms.CharField(required=False) @@ -480,6 +480,7 @@ def _state_to_doc_type(state): ).distinct(): if p in get_active_ads(): ads.append(p) + ads.sort(key=lambda p: normalize_for_sorting(p.plain_name())) bucket_template = { dt: {state: [[] for _ in range(days)] for state in STATE_SLUGS[dt].values()} diff --git a/ietf/utils/unicodenormalize.py b/ietf/utils/unicodenormalize.py new file mode 100644 index 0000000000..8644dbdb79 --- /dev/null +++ b/ietf/utils/unicodenormalize.py @@ -0,0 +1,9 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +import unicodedata + +def normalize_for_sorting(text): + """Normalize text for proper accent-aware sorting.""" + # Normalize the text to NFD (decomposed form) + decomposed = unicodedata.normalize('NFD', text) + # Filter out combining diacritical marks + return ''.join(char for char in decomposed if not unicodedata.combining(char)) From bc4752d8a09070680efe0f198fb4f8ac7b125233 Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Fri, 7 Nov 2025 14:47:08 -0500 Subject: [PATCH 010/158] fix: AD order in IESG ballots (and other places) does not handle UTF-8 collation (#9836) * Fix the ballot AD order by name * Also fix the IESG WG dashboard * Fix IESG milestones under review * Better normalization by removing completely the diacritical marks * Align comments * Test whether areas are alphabetically sorted --- ietf/doc/views_doc.py | 4 +- ietf/iesg/tests.py | 89 ++++++++++++++++++++++--------------------- ietf/iesg/utils.py | 7 ++-- ietf/iesg/views.py | 3 +- 4 files changed, 53 insertions(+), 50 deletions(-) diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 50070734f9..5210317325 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -92,7 +92,7 @@ from ietf.utils.response import permission_denied from ietf.utils.text import maybe_split from ietf.utils.timezone import date_today - +from ietf.utils.unicodenormalize import normalize_for_sorting def render_document_top(request, doc, tab, name): tabs = [] @@ -1506,7 +1506,7 @@ def document_ballot_content(request, doc, ballot_id, editable=True): position_groups = [] for n in BallotPositionName.objects.filter(slug__in=[p.pos_id for p in positions]).order_by('order'): g = (n, [p for p in positions if p.pos_id == n.slug]) - g[1].sort(key=lambda p: (p.is_old_pos, p.balloter.plain_name())) + g[1].sort(key=lambda p: (p.is_old_pos, normalize_for_sorting(p.balloter.plain_name()))) if n.blocking: position_groups.insert(0, g) else: diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py index f3778d1ded..e5fbe5da7b 100644 --- a/ietf/iesg/tests.py +++ b/ietf/iesg/tests.py @@ -281,12 +281,13 @@ def test_working_groups(self): wg_summary, ) = get_wg_dashboard_info() + # checks for the expected result with area sorted by name self.assertEqual( area_summary, [ { - "area": "red", - "groups_in_area": 1, + "area": "blue", + "groups_in_area": 3, "groups_with_docs": 0, "doc_count": 0, "page_count": 0, @@ -295,8 +296,8 @@ def test_working_groups(self): "page_percent": 0, }, { - "area": "orange", - "groups_in_area": 4, + "area": "green", + "groups_in_area": 3, "groups_with_docs": 0, "doc_count": 0, "page_count": 0, @@ -305,8 +306,8 @@ def test_working_groups(self): "page_percent": 0, }, { - "area": "yellow", - "groups_in_area": 2, + "area": "orange", + "groups_in_area": 4, "groups_with_docs": 0, "doc_count": 0, "page_count": 0, @@ -315,8 +316,8 @@ def test_working_groups(self): "page_percent": 0, }, { - "area": "green", - "groups_in_area": 3, + "area": "red", + "groups_in_area": 1, "groups_with_docs": 0, "doc_count": 0, "page_count": 0, @@ -325,8 +326,8 @@ def test_working_groups(self): "page_percent": 0, }, { - "area": "blue", - "groups_in_area": 3, + "area": "violet", + "groups_in_area": 4, "groups_with_docs": 0, "doc_count": 0, "page_count": 0, @@ -335,8 +336,8 @@ def test_working_groups(self): "page_percent": 0, }, { - "area": "violet", - "groups_in_area": 4, + "area": "yellow", + "groups_in_area": 2, "groups_with_docs": 0, "doc_count": 0, "page_count": 0, @@ -1192,34 +1193,14 @@ def test_working_groups(self): area_summary, [ { - "area": "red", - "groups_in_area": 1, - "groups_with_docs": 1, - "doc_count": 1, - "page_count": 7, - "group_percent": 6.25, - "doc_percent": 3.571428571428571, - "page_percent": 3.5897435897435894, - }, - { - "area": "orange", - "groups_in_area": 4, + "area": "blue", + "groups_in_area": 3, "groups_with_docs": 3, - "doc_count": 4, - "page_count": 29, + "doc_count": 6, + "page_count": 40, "group_percent": 18.75, - "doc_percent": 14.285714285714285, - "page_percent": 14.871794871794872, - }, - { - "area": "yellow", - "groups_in_area": 2, - "groups_with_docs": 2, - "doc_count": 2, - "page_count": 17, - "group_percent": 12.5, - "doc_percent": 7.142857142857142, - "page_percent": 8.717948717948717, + "doc_percent": 21.428571428571427, + "page_percent": 20.51282051282051, }, { "area": "green", @@ -1232,14 +1213,24 @@ def test_working_groups(self): "page_percent": 11.282051282051283, }, { - "area": "blue", - "groups_in_area": 3, + "area": "orange", + "groups_in_area": 4, "groups_with_docs": 3, - "doc_count": 6, - "page_count": 40, + "doc_count": 4, + "page_count": 29, "group_percent": 18.75, - "doc_percent": 21.428571428571427, - "page_percent": 20.51282051282051, + "doc_percent": 14.285714285714285, + "page_percent": 14.871794871794872, + }, + { + "area": "red", + "groups_in_area": 1, + "groups_with_docs": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 6.25, + "doc_percent": 3.571428571428571, + "page_percent": 3.5897435897435894, }, { "area": "violet", @@ -1251,6 +1242,16 @@ def test_working_groups(self): "doc_percent": 35.714285714285715, "page_percent": 41.02564102564102, }, + { + "area": "yellow", + "groups_in_area": 2, + "groups_with_docs": 2, + "doc_count": 2, + "page_count": 17, + "group_percent": 12.5, + "doc_percent": 7.142857142857142, + "page_percent": 8.717948717948717, + }, ], ) self.assertEqual( diff --git a/ietf/iesg/utils.py b/ietf/iesg/utils.py index 9051cf92b2..1d24ecac8e 100644 --- a/ietf/iesg/utils.py +++ b/ietf/iesg/utils.py @@ -12,7 +12,7 @@ from ietf.group.models import Group from ietf.iesg.agenda import get_doc_section from ietf.person.utils import get_active_ads - +from ietf.utils.unicodenormalize import normalize_for_sorting TelechatPageCount = namedtuple( "TelechatPageCount", @@ -192,6 +192,7 @@ def get_wg_dashboard_info(): else 0, } ) + area_summary.sort(key=lambda r: r["area"]) area_totals = { "group_count": groups_total, "doc_count": docs_total, @@ -238,7 +239,7 @@ def get_wg_dashboard_info(): else 0, } ) - noad_summary.sort(key=lambda r: (r["ad"], r["area"])) + noad_summary.sort(key=lambda r: (normalize_for_sorting(r["ad"]), r["area"])) ad_summary = [] ad_totals = { @@ -278,7 +279,7 @@ def get_wg_dashboard_info(): else 0, } ) - ad_summary.sort(key=lambda r: (r["ad"], r["area"])) + ad_summary.sort(key=lambda r: (normalize_for_sorting(r["ad"]), r["area"])) rfc_counter = Counter( Document.objects.filter(type="rfc").values_list("group__acronym", flat=True) diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index 014b290425..f03afb9fc1 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -68,6 +68,7 @@ from ietf.meeting.utils import get_activity_stats from ietf.doc.utils_search import fill_in_document_table_attributes, fill_in_telechat_date from ietf.utils.timezone import date_today, datetime_from_date +from ietf.utils.unicodenormalize import normalize_for_sorting def review_decisions(request, year=None): events = DocEvent.objects.filter(type__in=("iesg_disapproved", "iesg_approved")) @@ -547,7 +548,7 @@ def milestones_needing_review(request): ) return render(request, 'iesg/milestones_needing_review.html', - dict(ads=sorted(ad_list, key=lambda ad: ad.plain_name()),)) + dict(ads=sorted(ad_list, key=lambda ad: normalize_for_sorting(ad.plain_name())),)) def photos(request): roles = sorted(Role.objects.filter(group__type='area', group__state='active', name_id='ad'),key=lambda x: "" if x.group.acronym=="gen" else x.group.acronym) From c43092560c50363f59914205d059fec103350720 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Sun, 9 Nov 2025 10:51:51 -0500 Subject: [PATCH 011/158] fix: add resolver records when importing minutes (#9907) --- ietf/meeting/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index a4fe498152..2ab670a04d 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -5531,6 +5531,7 @@ def import_session_minutes(request, session_id, num): except SaveMaterialsError as err: form.add_error(None, str(err)) else: + resolve_uploaded_material(meeting=session.meeting, doc=session.minutes()) messages.success(request, f'Successfully imported minutes as revision {session.minutes().rev}.') return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym) else: From d252e85642261e8881e5fbec2d051e06c1616e07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Nov 2025 18:38:48 -0500 Subject: [PATCH 012/158] chore(deps): bump actions/setup-node from 4 to 6 (#9758) Bumps [actions/setup-node](https://github.com/actions/setup-node) from 4 to 6. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v4...v6) --- updated-dependencies: - dependency-name: actions/setup-node dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7eac7b1c64..24ab51ebcf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -170,7 +170,7 @@ jobs: fetch-tags: false - name: Setup Node.js environment - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: 18.x diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f10c1db9a3..836314bac0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -104,7 +104,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: node-version: '18' From 3f6addc6ebba12536d941b51c252168c9fc4f182 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 12 Nov 2025 10:30:43 -0600 Subject: [PATCH 013/158] fix: link to sdo group page from other sdo list (#9920) --- ietf/templates/liaisons/list_other_sdo.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/templates/liaisons/list_other_sdo.html b/ietf/templates/liaisons/list_other_sdo.html index 23f2d94af3..e6a567ae50 100644 --- a/ietf/templates/liaisons/list_other_sdo.html +++ b/ietf/templates/liaisons/list_other_sdo.html @@ -23,7 +23,7 @@

        {{sdos_in_state.grouper|capfirst}} SDO groups

        {{ group.acronym }}{{ group.acronym }} {{ group.name }} {% for person in group.liaison_managers %} From 8cabf4d6c41ce1a73602b5ac1e1d372086266f71 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 12 Nov 2025 14:55:21 -0600 Subject: [PATCH 014/158] fix(dev): Disable python terminal integration as it corrupts the terminal (#9924) --- .vscode/settings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index b0ceba5c9d..b323cd02f7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -56,5 +56,6 @@ "python.linting.pylintArgs": ["--load-plugins", "pylint_django"], "python.testing.pytestEnabled": false, "python.testing.unittestEnabled": false, - "python.linting.enabled": true + "python.linting.enabled": true, + "python.terminal.shellIntegration.enabled": false } From 92f23c0066c439ae7a696ddd8929418698cd9185 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 13 Nov 2025 09:36:52 -0600 Subject: [PATCH 015/158] fix: adjust test for other sdo list page (#9929) --- ietf/liaisons/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index 5e0a237575..e29045443f 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -119,7 +119,7 @@ def test_list_other_sdo(self): r = self.client.get(url) q = PyQuery(r.content) self.assertEqual(len(q("h1")), 2) - first_td_elements_text = [e.text for e in q("tr").find("td:first-child")] + first_td_elements_text = [e.text for e in q("tr").find("td:first-child a")] self.assertEqual(first_td_elements_text, ["first", "second", "third"]) class UnitTests(TestCase): From 9244e6107c4923fb6186126302bd6b71000680e3 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Fri, 14 Nov 2025 17:37:14 -0500 Subject: [PATCH 016/158] ci(staging): Increase wait-for-completion timeout to 60 minutes --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 24ab51ebcf..df218930ab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -453,7 +453,7 @@ jobs: token: ${{ secrets.GH_INFRA_K8S_TOKEN }} inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "waitClusterReady":true }' wait-for-completion: true - wait-for-completion-timeout: 30m + wait-for-completion-timeout: 60m wait-for-completion-interval: 20s display-workflow-run-url: false From 70067702da41414710151e176ddaf57a3bf8ef64 Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Tue, 18 Nov 2025 09:19:59 -0500 Subject: [PATCH 017/158] refactor: use ical for upcoming meetings (#9845) * refactor: use ical for upcoming meetings * fix: unfold to pass tests * fix: make material title, type and url on one line --- ietf/meeting/tests_views.py | 7 ++- ietf/meeting/views.py | 69 ++++++++++++++++++++++++----- ietf/templates/meeting/upcoming.ics | 32 ------------- 3 files changed, 64 insertions(+), 44 deletions(-) delete mode 100644 ietf/templates/meeting/upcoming.ics diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index bab2de5ec7..b94229d969 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -5302,7 +5302,9 @@ def test_upcoming_ical(self): assert_ical_response_is_valid(self, r, expected_event_summaries=expected_event_summaries, expected_event_count=len(expected_event_summaries)) - self.assertContains(r, 'Remote instructions: https://someurl.example.com') + # Unfold long lines that might have been folded by iCal + content_unfolded = r.content.decode('utf-8').replace('\r\n ', '') + self.assertIn('Remote instructions: https://someurl.example.com', content_unfolded) Session.objects.filter(meeting__type_id='interim').update(remote_instructions='') r = self.client.get(url) @@ -5310,7 +5312,8 @@ def test_upcoming_ical(self): assert_ical_response_is_valid(self, r, expected_event_summaries=expected_event_summaries, expected_event_count=len(expected_event_summaries)) - self.assertNotContains(r, 'Remote instructions:') + content_unfolded = r.content.decode('utf-8').replace('\r\n ', '') + self.assertNotIn('Remote instructions:', content_unfolded) updated = meeting.updated() self.assertIsNotNone(updated) diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 2ab670a04d..903e3c7e79 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -4430,16 +4430,7 @@ def upcoming_ical(request): else: ietfs = [] - meeting_vtz = {meeting.vtimezone() for meeting in meetings} - meeting_vtz.discard(None) - - # icalendar response file should have '\r\n' line endings per RFC5545 - response = render_to_string('meeting/upcoming.ics', { - 'vtimezones': ''.join(sorted(meeting_vtz)), - 'assignments': assignments, - 'ietfs': ietfs, - }, request=request) - response = parse_ical_line_endings(response) + response = render_upcoming_ical(assignments, ietfs, request) response = HttpResponse(response, content_type='text/calendar') response['Content-Disposition'] = 'attachment; filename="upcoming.ics"' @@ -4513,6 +4504,64 @@ def render_important_dates_ical(meetings, request): return cal.to_ical().decode("utf-8") +def render_upcoming_ical(assignments, meetings, request): + """Generate upcoming using the icalendar library""" + + cal = Calendar() + cal.add("prodid", "-//IETF//datatracker.ietf.org ical upcoming//EN") + cal.add("version", "2.0") + cal.add("method", "PUBLISH") + + for item in assignments: + event = Event() + + event.add("uid", f"ietf-{item.session.meeting.number}-{item.timeslot.pk}") + event.add("summary", f"{item.session.group.acronym.lower()} - {item.session.name if item.session.name else item.session.group.name}") + + if item.schedule.meeting.city: + event.add("location", f"{item.schedule.meeting.city},{item.schedule.meeting.country}") + + event.add("status", item.session.ical_status) + event.add("class", "PUBLIC") + + event.add("dtstart", item.timeslot.utc_start_time()) + event.add("dtend", item.timeslot.utc_end_time()) + event.add("dtstamp", item.timeslot.modified) + if item.session.agenda(): + event.add("url", item.session.agenda().get_href()) + + description_lines = [] + if item.timeslot.name: + description_lines.append(f"{item.timeslot.name}") + if item.session.agenda_note: + description_lines.append(f"Note: {item.session.agenda_note}") + + for material in item.session.materials.all(): + title_part = f" ({material.title})" if material.type.name != "Agenda" else "" + description_lines.append(f"{material.type}{title_part}: {material.get_href()}") + + if item.session.remote_instructions: + description_lines.append(f"Remote instructions: {item.session.remote_instructions}") + + event.add("description", "\n".join(description_lines)) + cal.add_component(event) + + for meeting in meetings: + event = Event() + event.add("uid", f"ietf-{meeting.number}") + event.add("summary", f"IETF {meeting.number}") + if meeting.city: + event.add("location", f"{meeting.city},{meeting.country}") + event.add("class", "PUBLIC") + event.add("dtstart", meeting.date) + event.add("dtend", meeting.end_date() + datetime.timedelta(days=1)) + event.add("dtstamp", meeting.cached_updated) + event.add("url", f"{request.scheme}://{request.get_host()}{reverse('agenda', kwargs={'num': meeting.number})}") + + cal.add_component(event) + + return cal.to_ical().decode("utf-8") + def upcoming_json(request): '''Return Upcoming meetings in json format''' today = date_today() diff --git a/ietf/templates/meeting/upcoming.ics b/ietf/templates/meeting/upcoming.ics deleted file mode 100644 index 5eca7ec81d..0000000000 --- a/ietf/templates/meeting/upcoming.ics +++ /dev/null @@ -1,32 +0,0 @@ -{% load humanize tz %}{% autoescape off %}{% load ietf_filters textfilters %}BEGIN:VCALENDAR -VERSION:2.0 -METHOD:PUBLISH -PRODID:-//IETF//datatracker.ietf.org ical upcoming//EN -{{vtimezones}}{% for item in assignments %}BEGIN:VEVENT -UID:ietf-{{item.session.meeting.number}}-{{item.timeslot.pk}} -SUMMARY:{% if item.session.name %}{{item.session.group.acronym|lower}} - {{item.session.name|ics_esc}}{% else %}{{item.session.group.acronym|lower}} - {{item.session.group.name}}{%endif%} -{% if item.schedule.meeting.city %}LOCATION:{{item.schedule.meeting.city}},{{item.schedule.meeting.country}} -{% endif %}STATUS:{{item.session.ical_status}} -CLASS:PUBLIC -DTSTART{% ics_date_time item.timeslot.local_start_time item.schedule.meeting.time_zone %} -DTEND{% ics_date_time item.timeslot.local_end_time item.schedule.meeting.time_zone %} -DTSTAMP{% ics_date_time item.timeslot.modified|utc 'utc' %}{% if item.session.agenda %} -URL:{{item.session.agenda.get_href}}{% endif %} -DESCRIPTION:{% if item.timeslot.name %}{{item.timeslot.name|ics_esc}}\n{% endif %}{% if item.session.agenda_note %} - Note: {{item.session.agenda_note|ics_esc}}\n{% endif %}{% for material in item.session.materials.all %} - \n{{material.type}}{% if material.type.name != "Agenda" %} - ({{material.title|ics_esc}}){% endif %}: - {{material.get_href}}\n{% endfor %}{% if item.session.remote_instructions %} - Remote instructions: {{ item.session.remote_instructions }}\n{% endif %} -END:VEVENT -{% endfor %}{% for meeting in ietfs %}BEGIN:VEVENT -UID:ietf-{{ meeting.number }} -SUMMARY:IETF {{ meeting.number }}{% if meeting.city %} -LOCATION:{{ meeting.city }},{{ meeting.country }}{% endif %} -CLASS:PUBLIC -DTSTART;VALUE=DATE{% if meeting.time_zone %};TZID={{ meeting.time_zone|ics_esc }}{% endif %}:{{ meeting.date|date:"Ymd" }} -DTEND;VALUE=DATE{% if meeting.time_zone %};TZID={{ meeting.time_zone|ics_esc }}{% endif %}:{{ meeting.end_date|next_day|date:"Ymd" }} -DTSTAMP{% ics_date_time meeting.cached_updated|utc 'utc' %} -URL:{{ request.scheme }}://{{ request.get_host }}{% url 'agenda' num=meeting.number %} -END:VEVENT -{% endfor %}END:VCALENDAR{% endautoescape %} From 33c0a894b3c1a4122f128117c295709e6f3c83ca Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 18 Nov 2025 10:01:16 -0600 Subject: [PATCH 018/158] test: fix selenium selector syntax (#9963) --- ietf/meeting/tests_js.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py index 262b47652c..3269342924 100644 --- a/ietf/meeting/tests_js.py +++ b/ietf/meeting/tests_js.py @@ -1260,7 +1260,7 @@ def _assert_ietf_tz_correct(meetings, tz): # to inherit Django's settings.TIME_ZONE but I don't know whether that's guaranteed to be consistent. # To avoid test fragility, ask Moment what it considers local and expect that. local_tz = self.driver.execute_script('return moment.tz.guess();') - local_tz_opt = tz_select_input.find_element(By.CSS_SELECTOR, 'option[value=%s]' % local_tz) + local_tz_opt = tz_select_input.find_element(By.CSS_SELECTOR, 'option[value="%s"]' % local_tz) local_tz_bottom_opt = tz_select_bottom_input.find_element(By.CSS_SELECTOR, 'option[value="%s"]' % local_tz) # Should start off in local time zone From 0dd84be354aa872e0bc2582295a97f7491dc7b21 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 18 Nov 2025 13:28:30 -0400 Subject: [PATCH 019/158] fix: recognize Editorial Stream in queue2.xml (#9965) --- ietf/sync/rfceditor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py index 76357ffbcc..b3234a87e2 100644 --- a/ietf/sync/rfceditor.py +++ b/ietf/sync/rfceditor.py @@ -115,6 +115,8 @@ def parse_queue(response): stream = "irtf" elif name.startswith("INDEPENDENT"): stream = "ise" + elif name.startswith("Editorial Stream"): + stream = "editorial" else: stream = None warnings.append("unrecognized section " + name) From a007f50518d027e5edcf2325a4b6de1c347a1a02 Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Tue, 18 Nov 2025 16:53:51 -0500 Subject: [PATCH 020/158] fix(dev): remove deprecated ext (#9966) --- .devcontainer/devcontainer.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index bf28550084..2cfff78853 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -33,7 +33,6 @@ "oderwat.indent-rainbow", "redhat.vscode-yaml", "spmeesseman.vscode-taskexplorer", - "visualstudioexptteam.vscodeintellicode", "ms-python.pylint", "charliermarsh.ruff" ], From c79b35dd1ddbdadd78481023fa6e6dd876a3db3c Mon Sep 17 00:00:00 2001 From: Absit Iniuria Date: Tue, 25 Nov 2025 20:12:18 +0000 Subject: [PATCH 021/158] fix: make chatlog readable in dark mode (#9995) --- client/components/ChatLog.vue | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/client/components/ChatLog.vue b/client/components/ChatLog.vue index f9dc382bfe..b3a4f7b40f 100644 --- a/client/components/ChatLog.vue +++ b/client/components/ChatLog.vue @@ -159,4 +159,18 @@ onMounted(() => { } } } + +[data-bs-theme="dark"] .chatlog { + .n-timeline-item-content__title { + color: #d63384 !important; + } + + .n-timeline-item-content__content { + color: #fff !important; + } + + .n-timeline-item-content__meta { + color: #0569ffd9 !important; + } +} From 3afa3d0e79fe98dd8f1780a6649340acbf0f317b Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Wed, 26 Nov 2025 00:05:07 -0500 Subject: [PATCH 022/158] ci: enable restoreToLastFullSnapshot for staging db deploy --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index df218930ab..5e91445202 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -451,7 +451,7 @@ jobs: repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "waitClusterReady":true }' + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }' wait-for-completion: true wait-for-completion-timeout: 60m wait-for-completion-interval: 20s From 03e84773497715d5e082f8b08d8aaa90e45d2a73 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 1 Dec 2025 11:02:08 -0400 Subject: [PATCH 023/158] chore: drop apt-key + old pg utilities (#10029) * chore: refactor to drop apt-key + consistency Updates postgres apt setup to use current recommendation + not use apt-key. Moves all installed keyrings to /etc/apt/keyrings instead of /usr/share/keyrings. Either is a reasonable place, but let's just use one. * chore: no duplicate key install, drop pg client 14 * chore: drop pgloader, too --- docker/app.Dockerfile | 5 ----- docker/base.Dockerfile | 27 ++++++++++++++------------- docker/celery.Dockerfile | 5 ----- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index fee3833733..dd4cf72ffd 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -10,12 +10,7 @@ ARG USER_GID=$USER_UID COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh -# Add Postgresql Apt Repository to get 14 -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 && apt-get purge -y imagemagick imagemagick-6-common \ # Install common packages, non-root user diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index c1fe5b093e..2501636049 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -11,21 +11,22 @@ RUN apt-get update \ # Add Node.js Source RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \ - && mkdir -p /etc/apt/keyrings\ - && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg -RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list -RUN echo "Package: nodejs" >> /etc/apt/preferences.d/preferences && \ - echo "Pin: origin deb.nodesource.com" >> /etc/apt/preferences.d/preferences && \ - echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/preferences + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list +RUN echo "Package: nodejs" >> /etc/apt/preferences.d/preferences \ + && echo "Pin: origin deb.nodesource.com" >> /etc/apt/preferences.d/preferences \ + && echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/preferences # Add Docker Source -RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg -RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null - -# Add PostgreSQL Source -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +RUN mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list + +# Add PostgreSQL Source +RUN mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/keyrings/apt.postgresql.org.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/apt.postgresql.org.gpg] https://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list # Install the packages we need RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends \ diff --git a/docker/celery.Dockerfile b/docker/celery.Dockerfile index e7c7b9cc3f..e93ca3cf77 100644 --- a/docker/celery.Dockerfile +++ b/docker/celery.Dockerfile @@ -10,12 +10,7 @@ ARG USER_GID=$USER_UID COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh -# Add Postgresql Apt Repository to get 14 -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 && apt-get purge -y imagemagick imagemagick-6-common \ # Install common packages, non-root user From 47c15df84ca8a3f401239a41214b66839b16ecc3 Mon Sep 17 00:00:00 2001 From: Absit Iniuria Date: Mon, 1 Dec 2025 15:13:37 +0000 Subject: [PATCH 024/158] fix: render polls correctly in darkmode (#10027) --- client/components/Polls.vue | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/client/components/Polls.vue b/client/components/Polls.vue index 30cc9e8f36..0846d4ed16 100644 --- a/client/components/Polls.vue +++ b/client/components/Polls.vue @@ -90,3 +90,21 @@ onMounted(() => { }) + From 9d10995dbc6d84bcce7712dc2b2b849286925223 Mon Sep 17 00:00:00 2001 From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com> Date: Mon, 1 Dec 2025 16:02:38 +0000 Subject: [PATCH 025/158] ci: update base image target version to 20251201T1548 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index d3b186e1f5..ae59ba1440 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20250903T2216 +FROM ghcr.io/ietf-tools/datatracker-app-base:20251201T1548 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 9d8427efdb..726f080c67 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20250903T2216 +20251201T1548 From ef4e0958e40380954577d9a4026a12bb9f8bf99e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 2 Dec 2025 19:07:29 -0400 Subject: [PATCH 026/158] fix: adjust patch for Django 4.2.27 (#10045) --- patch/django-cookie-delete-with-all-settings.patch | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/patch/django-cookie-delete-with-all-settings.patch b/patch/django-cookie-delete-with-all-settings.patch index fb8bbbe4fe..4ceaf8fceb 100644 --- a/patch/django-cookie-delete-with-all-settings.patch +++ b/patch/django-cookie-delete-with-all-settings.patch @@ -9,9 +9,9 @@ samesite=settings.SESSION_COOKIE_SAMESITE, ) ---- django/http/response.py.orig 2020-08-13 11:16:04.060627793 +0200 -+++ django/http/response.py 2020-08-13 11:54:03.482476973 +0200 -@@ -282,20 +282,28 @@ +--- django/http/response.py.orig 2025-12-02 22:12:05.197283001 +0000 ++++ django/http/response.py 2025-12-02 22:26:01.396576013 +0000 +@@ -286,20 +286,28 @@ value = signing.get_cookie_signer(salt=key + salt).sign(value) return self.set_cookie(key, value, **kwargs) From 9a4ad7223155814cc8cf8e7b6c4e581b37e7d119 Mon Sep 17 00:00:00 2001 From: Absit Iniuria Date: Mon, 15 Dec 2025 14:46:12 +0000 Subject: [PATCH 027/158] fix: adjust datepicker css in darkmode (#10095) --- ietf/static/css/datepicker.scss | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/ietf/static/css/datepicker.scss b/ietf/static/css/datepicker.scss index 88f9e835fd..b193ccda3a 100644 --- a/ietf/static/css/datepicker.scss +++ b/ietf/static/css/datepicker.scss @@ -4,3 +4,29 @@ $dp-cell-focus-background-color: $dropdown-link-hover-bg !default; @import "vanillajs-datepicker/sass/datepicker-bs5"; + +[data-bs-theme="dark"] .datepicker-picker { + .datepicker-header, + .datepicker-controls .btn, + .datepicker-main, + .datepicker-footer { + background-color: $gray-800; + } + + .datepicker-cell:hover { + background-color: $gray-700; + } + + .datepicker-cell.day.focused { + background-color: $gray-600; + } + + .datepicker-cell.day.selected.focused { + background-color: $blue; + } + + .datepicker-controls .btn:hover { + background-color:$gray-700; + color: $gray-400; + } +} From e775af342a3ff7a90d33166ac653ec99b8952d4e Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 16 Dec 2025 09:10:56 -0600 Subject: [PATCH 028/158] feat: editable wglc messages (#9607) (#9616) * feat: editable wglc messages (#9607) * feat: split complex actions out of change state view * feat: allow editing call for adoption messages (#9702) * feat: (WIP) sketch of workflow changes to allow editing call for adoption message * feat: improved tests and related code adjustments * feat: improve the change state interstitial workflow * fix: simplify call for adoption workflow (#9964) * fix: simplify call for adoption workflow * fix: guard against nonwg target groups * refactor: make reject conditions easier to understand * feat: add issuewglc form automatic date replacement (#9969) * fix: add date adjusting js to the wglc email form (#9976) * fix: populate and use cfa and wglc forms correctly (#9981) * fix: populate cfa form correctly. use cfa form for email. * fix: restrict wglc to drafts * fix: use wglc form for email. * fix: tweak wording on forms and email (#9998) * fix: adjust when call for adoption is shown, allow direct setting of c-adopt and wg-lc (#10051) * fix: adjust when to show the call for adoption button * fix: allow direct setting of c-adopt and wg-lc * fix: adjust call for adoption wording (#10076) --------- Co-authored-by: Nicolas Giard --- ietf/doc/mails.py | 55 -- ietf/doc/templatetags/ietf_filters.py | 58 ++ ietf/doc/tests_draft.py | 501 ++++++++++++++---- ietf/doc/urls.py | 5 + ietf/doc/views_doc.py | 8 +- ietf/doc/views_draft.py | 386 +++++++++++++- ietf/ietfauth/utils.py | 4 +- ietf/templates/doc/document_draft.html | 7 +- .../doc/draft/ask_about_ietf_adoption.html | 22 + .../doc/draft/change_stream_state.html | 19 +- ...issue_working_group_call_for_adoption.html | 61 +++ .../draft/issue_working_group_last_call.html | 61 +++ .../doc/draft/wg_action_helpers.html | 25 + .../doc/mail/wg_call_for_adoption_issued.txt | 24 +- .../doc/mail/wg_last_call_issued.txt | 22 +- 15 files changed, 1066 insertions(+), 192 deletions(-) create mode 100644 ietf/templates/doc/draft/ask_about_ietf_adoption.html create mode 100644 ietf/templates/doc/draft/issue_working_group_call_for_adoption.html create mode 100644 ietf/templates/doc/draft/issue_working_group_last_call.html create mode 100644 ietf/templates/doc/draft/wg_action_helpers.html diff --git a/ietf/doc/mails.py b/ietf/doc/mails.py index f20d398c3c..ddecbb6b54 100644 --- a/ietf/doc/mails.py +++ b/ietf/doc/mails.py @@ -103,61 +103,6 @@ def email_stream_changed(request, doc, old_stream, new_stream, text=""): dict(text=text, url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url()), cc=cc) - -def email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=None): - if cfa_duration_weeks is None: - cfa_duration_weeks=2 - (to, cc) = gather_address_lists("doc_wg_call_for_adoption_issued", doc=doc) - frm = request.user.person.formatted_email() - - end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=7 * cfa_duration_weeks) - - subject = f"Call for adoption: {doc.name}-{doc.rev} (Ends {end_date})" - - send_mail( - request, - to, - frm, - subject, - "doc/mail/wg_call_for_adoption_issued.txt", - dict( - doc=doc, - subject=subject, - url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(), - end_date=end_date, - cfa_duration_weeks=cfa_duration_weeks, - wg_list=doc.group.list_email, - ), - cc=cc, - ) - - -def email_wg_last_call_issued(request, doc, wglc_duration_weeks=None): - if wglc_duration_weeks is None: - wglc_duration_weeks = 2 - (to, cc) = gather_address_lists("doc_wg_last_call_issued", doc=doc) - frm = request.user.person.formatted_email() - - - end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=7 * wglc_duration_weeks) - subject = f"WG Last Call: {doc.name}-{doc.rev} (Ends {end_date})" - - send_mail( - request, - to, - frm, - subject, - "doc/mail/wg_last_call_issued.txt", - dict( - doc=doc, - subject=subject, - url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(), - end_date=end_date, - wglc_duration_weeks=wglc_duration_weeks, - wg_list=doc.group.list_email, - ), - cc=cc, - ) def email_pulled_from_rfc_queue(request, doc, comment, prev_state, next_state): extra=extra_automation_headers(doc) diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index 5cabe1728d..ae5df641c2 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -1017,3 +1017,61 @@ def is_in_stream(doc): elif stream == "editorial": return True return False + + +@register.filter +def is_doc_ietf_adoptable(doc): + return doc.stream_id is None or all( + [ + doc.stream_id == "ietf", + doc.get_state_slug("draft-stream-ietf") + not in [ + "c-adopt", + "adopt-wg", + "info", + "wg-doc", + "parked", + "dead", + "wg-lc", + "waiting-for-implementation", + "chair-w", + "writeupw", + "sub-pub", + ], + doc.get_state_slug("draft") != "rfc", + doc.became_rfc() is None, + ] + ) + + +@register.filter +def can_issue_ietf_wg_lc(doc): + return all( + [ + doc.stream_id == "ietf", + doc.get_state_slug("draft-stream-ietf") + not in ["wg-cand", "c-adopt", "wg-lc"], + doc.get_state_slug("draft") != "rfc", + doc.became_rfc() is None, + ] + ) + + +@register.filter +def can_submit_to_iesg(doc): + return all( + [ + doc.stream_id == "ietf", + doc.get_state_slug("draft-iesg") == "idexists", + doc.get_state_slug("draft-stream-ietf") not in ["wg-cand", "c-adopt"], + ] + ) + + +@register.filter +def has_had_ietf_wg_lc(doc): + return ( + doc.stream_id == "ietf" + and doc.docevent_set.filter(statedocevent__state__slug="wg-lc").exists() + ) + diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index 4d262c5a2f..cebeac1f27 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -21,13 +21,14 @@ import debug # pyflakes:ignore from ietf.doc.expire import expirable_drafts, get_expired_drafts, send_expire_notice_for_draft, expire_draft -from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, WgDraftFactory, RgDraftFactory, DocEventFactory +from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, StateDocEventFactory, WgDraftFactory, RgDraftFactory, DocEventFactory, WgRfcFactory from ietf.doc.models import ( Document, DocReminder, DocEvent, ConsensusDocEvent, LastCallDocEvent, RelatedDocument, State, TelechatDocEvent, WriteupDocEvent, DocRelationshipName, IanaExpertDocEvent ) from ietf.doc.storage_utils import exists_in_storage, store_str from ietf.doc.utils import get_tags_for_stream_id, create_ballot_if_not_open -from ietf.doc.views_draft import AdoptDraftForm +from ietf.doc.views_draft import AdoptDraftForm, IssueCallForAdoptionForm, IssueWorkingGroupLastCallForm +from ietf.ietfauth.utils import has_role from ietf.name.models import DocTagName, RoleName from ietf.group.factories import GroupFactory, RoleFactory from ietf.group.models import Group, Role @@ -86,7 +87,7 @@ def test_ad_approved(self): self.assertTrue("Approved: " in outbox[-1]['Subject']) self.assertTrue(draft.name in outbox[-1]['Subject']) self.assertTrue('iesg@' in outbox[-1]['To']) - + def test_change_state(self): ad = Person.objects.get(user__username="ad") draft = WgDraftFactory( @@ -1708,11 +1709,7 @@ def test_adopt_document(self): self.assertEqual(draft.group, chair_role.group) self.assertEqual(draft.stream_id, stream_state_type_slug[type_id][13:]) # trim off "draft-stream-" self.assertEqual(draft.docevent_set.count() - events_before, 5) - self.assertEqual(len(outbox), 2) - self.assertTrue("Call For Adoption" in outbox[0]["Subject"]) - self.assertTrue(f"{chair_role.group.acronym}-chairs@" in outbox[0]['To']) - self.assertTrue(f"{draft.name}@" in outbox[0]['To']) - self.assertTrue(f"{chair_role.group.acronym}@" in outbox[0]['To']) + self.assertEqual(len(outbox), 1) # contents of outbox[1] are tested elsewhere # adopt @@ -2003,6 +2000,40 @@ def test_set_state(self): self.assertTrue("mars-chairs@ietf.org" in outbox[0].as_string()) self.assertTrue("marsdelegate@ietf.org" in outbox[0].as_string()) + def test_set_stream_state_to_wglc(self): + def _form_presents_state_option(response, state): + q = PyQuery(response.content) + option = q(f"select#id_new_state option[value='{state.pk}']") + return len(option) != 0 + + doc = WgDraftFactory() + chair = RoleFactory(name_id="chair", group=doc.group).person + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=doc.name, state_type="draft-stream-ietf"), + ) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.get(url) + wglc_state = State.objects.get(type="draft-stream-ietf", slug="wg-lc") + doc.set_state(wglc_state) + StateDocEventFactory( + doc=doc, + state_type_id="draft-stream-ietf", + state=("draft-stream-ietf", "wg-lc"), + ) + self.assertEqual(doc.docevent_set.count(), 2) + r = self.client.get(url) + self.assertTrue(_form_presents_state_option(r, wglc_state)) + other_doc = WgDraftFactory() + self.client.logout() + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=other_doc.name, state_type="draft-stream-ietf"), + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertTrue(_form_presents_state_option(r, wglc_state)) + def test_wg_call_for_adoption_issued(self): role = RoleFactory( name_id="chair", @@ -2029,12 +2060,7 @@ def test_wg_call_for_adoption_issued(self): ), ) self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("Call for adoption", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 10-week", body) + self.assertEqual(len(outbox), 1) # Test not entering a duration on the form draft = IndividualDraftFactory() url = urlreverse( @@ -2051,12 +2077,7 @@ def test_wg_call_for_adoption_issued(self): ), ) self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("Call for adoption", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 2-week", body) + self.assertEqual(len(outbox), 1) # Test the less usual workflow of issuing a call for adoption # of a document that's already in the ietf stream @@ -2086,12 +2107,7 @@ def test_wg_call_for_adoption_issued(self): ), ) self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("Call for adoption", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 10-week", body) + self.assertEqual(len(outbox), 1) draft = WgDraftFactory(group=role.group) url = urlreverse( "ietf.doc.views_draft.change_stream_state", @@ -2117,85 +2133,210 @@ def test_wg_call_for_adoption_issued(self): ), ) self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("Call for adoption", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 2-week", body) + self.assertEqual(len(outbox), 1) - def test_wg_last_call_issued(self): - role = RoleFactory( - name_id="chair", - group__acronym="mars", - group__list_email="mars-wg@ietf.org", - person__user__username="marschairman", - person__name="WG Cháir Man", + def test_issue_wg_lc_form(self): + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=1) + post = dict( + end_date=end_date, + to="foo@example.net, bar@example.com", + # Intentionally not passing cc + subject=f"garbage {end_date.isoformat()}", + body=f"garbage {end_date.isoformat()}", ) - draft = WgDraftFactory(group=role.group) - url = urlreverse( - "ietf.doc.views_draft.change_stream_state", - kwargs=dict(name=draft.name, state_type="draft-stream-ietf"), + form = IssueWorkingGroupLastCallForm(post) + self.assertTrue(form.is_valid()) + post["end_date"] = date_today(DEADLINE_TZINFO) + form = IssueWorkingGroupLastCallForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + "End date must be later than today", + form.errors["end_date"], + "Form accepted a too-early date", ) - login_testing_unauthorized(self, "marschairman", url) - old_state = draft.get_state("draft-stream-%s" % draft.stream_id) - new_state = State.objects.get( - used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-lc" + post["end_date"] = end_date + datetime.timedelta(days=2) + form = IssueWorkingGroupLastCallForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + f"Last call end date ({post['end_date'].isoformat()}) not found in subject", + form.errors["subject"], + "form allowed subject without end_date", ) - self.assertNotEqual(old_state, new_state) + self.assertIn( + f"Last call end date ({post['end_date'].isoformat()}) not found in body", + form.errors["body"], + "form allowed body without end_date", + ) + + def test_issue_wg_lc(self): + def _assert_rejected(testcase, doc, person): + url = urlreverse( + "ietf.doc.views_draft.issue_wg_lc", kwargs=dict(name=doc.name) + ) + login_testing_unauthorized(testcase, person.user.username, url) + r = testcase.client.get(url) + testcase.assertEqual(r.status_code, 404) + testcase.client.logout() + + already_rfc = WgDraftFactory(states=[("draft", "rfc")]) + rfc_chair = RoleFactory(name_id="chair", group=already_rfc.group).person + _assert_rejected(self, already_rfc, rfc_chair) + rg_doc = RgDraftFactory() + rg_chair = RoleFactory(name_id="chair", group=rg_doc.group).person + _assert_rejected(self, rg_doc, rg_chair) + inwglc_doc = WgDraftFactory(states=[("draft-stream-ietf", "wg-lc")]) + inwglc_chair = RoleFactory(name_id="chair", group=inwglc_doc.group).person + _assert_rejected(self, inwglc_doc, inwglc_chair) + doc = WgDraftFactory() + chair = RoleFactory(name_id="chair", group=doc.group).person + url = urlreverse("ietf.doc.views_draft.issue_wg_lc", kwargs=dict(name=doc.name)) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + postdict = dict() + postdict["end_date"] = q("input#id_end_date").attr("value") + postdict["to"] = q("input#id_to").attr("value") + ", extrato@example.org" + cc = q("input#id_cc").attr("value") + if cc is not None: + postdict["cc"] = cc + ", extracc@example.org" + else: + postdict["cc"] = "extracc@example.org" + postdict["subject"] = q("input#id_subject").attr("value") + " Extra Subject Words" + postdict["body"] = q("textarea#id_body").text() + "FGgqbQ$UNeXs" empty_outbox() r = self.client.post( url, - dict( - new_state=new_state.pk, - comment="some comment", - weeks="10", - tags=[ - t.pk - for t in draft.tags.filter( - slug__in=get_tags_for_stream_id(draft.stream_id) - ) - ], - ), + postdict, ) self.assertEqual(r.status_code, 302) + self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "wg-lc") self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) + self.assertIn(f"{doc.group.acronym}@ietf.org", outbox[1]["To"]) + self.assertIn("extrato@example.org", outbox[1]["To"]) + self.assertIn("extracc@example.org", outbox[1]["Cc"]) + self.assertIn("Extra Subject Words", outbox[1]["Subject"]) self.assertIn("WG Last Call", outbox[1]["Subject"]) body = get_payload_text(outbox[1]) self.assertIn("disclosure obligations", body) - self.assertIn("starts a 10-week", body) - draft = WgDraftFactory(group=role.group) - url = urlreverse( - "ietf.doc.views_draft.change_stream_state", - kwargs=dict(name=draft.name, state_type="draft-stream-ietf"), + self.assertIn("FGgqbQ$UNeXs", body) + + def test_issue_wg_call_for_adoption_form(self): + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=1) + post = dict( + end_date=end_date, + to="foo@example.net, bar@example.com", + # Intentionally not passing cc + subject=f"garbage {end_date.isoformat()}", + body=f"garbage {end_date.isoformat()}", ) - old_state = draft.get_state("draft-stream-%s" % draft.stream_id) - new_state = State.objects.get( - used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-lc" + form = IssueCallForAdoptionForm(post) + self.assertTrue(form.is_valid()) + post["end_date"] = date_today(DEADLINE_TZINFO) + form = IssueCallForAdoptionForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + "End date must be later than today", + form.errors["end_date"], + "Form accepted a too-early date", ) - self.assertNotEqual(old_state, new_state) - empty_outbox() - r = self.client.post( - url, - dict( - new_state=new_state.pk, - comment="some comment", - tags=[ - t.pk - for t in draft.tags.filter( - slug__in=get_tags_for_stream_id(draft.stream_id) - ) - ], - ), + post["end_date"] = end_date + datetime.timedelta(days=2) + form = IssueCallForAdoptionForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + f"Call for adoption end date ({post['end_date'].isoformat()}) not found in subject", + form.errors["subject"], + "form allowed subject without end_date", + ) + self.assertIn( + f"Call for adoption end date ({post['end_date'].isoformat()}) not found in body", + form.errors["body"], + "form allowed body without end_date", + ) + + def test_issue_wg_call_for_adoption(self): + def _assert_rejected(testcase, doc, person, group=None): + target_acronym = group.acronym if group is not None else doc.group.acronym + url = urlreverse( + "ietf.doc.views_draft.issue_wg_call_for_adoption", + kwargs=dict(name=doc.name, acronym=target_acronym), + ) + login_testing_unauthorized(testcase, person.user.username, url) + r = testcase.client.get(url) + testcase.assertEqual(r.status_code, 403) + testcase.client.logout() + + def _verify_call_issued(testcase, doc, chair_role): + url = urlreverse( + "ietf.doc.views_draft.issue_wg_call_for_adoption", + kwargs=dict(name=doc.name, acronym=chair_role.group.acronym), + ) + login_testing_unauthorized(testcase, chair_role.person.user.username, url) + r = testcase.client.get(url) + testcase.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + postdict = dict() + postdict["end_date"] = q("input#id_end_date").attr("value") + postdict["to"] = q("input#id_to").attr("value") + ", extrato@example.com" + self.assertIn(chair_role.group.list_email, postdict["to"]) + cc = q("input#id_cc").attr("value") + if cc is not None: + postdict["cc"] = cc + ", extracc@example.com" + else: + postdict["cc"] = "extracc@example.com" + postdict["subject"] = q("input#id_subject").attr("value") + " Extra Subject Words" + postdict["body"] = q("textarea#id_body").text() + "FGgqbQ$UNeXs" + empty_outbox() + r = testcase.client.post( + url, + postdict, + ) + testcase.assertEqual(r.status_code, 302) + doc.refresh_from_db() + self.assertEqual(doc.group, chair_role.group) + self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "c-adopt") + self.assertEqual(len(outbox), 2) + self.assertIn(f"{doc.group.acronym}@ietf.org", outbox[1]["To"]) + self.assertIn("extrato@example.com", outbox[1]["To"]) + self.assertIn("extracc@example.com", outbox[1]["Cc"]) + self.assertIn("Call for adoption", outbox[1]["Subject"]) + self.assertIn("Extra Subject Words", outbox[1]["Subject"]) + body = get_payload_text(outbox[1]) + self.assertIn("disclosure obligations", body) + self.assertIn("FGgqbQ$UNeXs", body) + self.client.logout() + return doc + + already_rfc = WgDraftFactory(states=[("draft", "rfc")]) + rfc = WgRfcFactory(group=already_rfc.group) + already_rfc.relateddocument_set.create(relationship_id="became_rfc",target=rfc) + rfc_chair = RoleFactory(name_id="chair", group=already_rfc.group).person + _assert_rejected(self, already_rfc, rfc_chair) + rg_doc = RgDraftFactory() + rg_chair = RoleFactory(name_id="chair", group=rg_doc.group).person + _assert_rejected(self, rg_doc, rg_chair) + inwglc_doc = WgDraftFactory(states=[("draft-stream-ietf", "wg-lc")]) + inwglc_chair = RoleFactory(name_id="chair", group=inwglc_doc.group).person + _assert_rejected(self, inwglc_doc, inwglc_chair) + ind_doc = IndividualDraftFactory() + _assert_rejected(self, ind_doc, rg_chair, rg_doc.group) + + # Successful call issued for doc already in WG + doc = WgDraftFactory(states=[("draft-stream-ietf","wg-cand")]) + chair_role = RoleFactory(name_id="chair",group=doc.group) + _ = _verify_call_issued(self, doc, chair_role) + + # Successful call issued for doc not yet in WG + doc = IndividualDraftFactory() + chair_role = RoleFactory(name_id="chair",group__type_id="wg") + doc = _verify_call_issued(self, doc, chair_role) + self.assertEqual(doc.group, chair_role.group) + self.assertEqual(doc.stream_id, "ietf") + self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "c-adopt") + self.assertCountEqual( + doc.docevent_set.values_list("type", flat=True), + ["changed_state", "changed_group", "changed_stream", "new_revision"] ) - self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("WG Last Call", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 2-week", body) def test_pubreq_validation(self): role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man') @@ -2393,6 +2534,188 @@ def test_editorial_metadata(self): self.assertNotIn("IESG", top_level_metadata_headings) self.assertNotIn("IANA", top_level_metadata_headings) +class IetfGroupActionHelperTests(TestCase): + def test_manage_adoption_routing(self): + draft = IndividualDraftFactory() + nobody = PersonFactory() + rgchair = RoleFactory(group__type_id="rg", name_id="chair").person + wgchair = RoleFactory(group__type_id="wg", name_id="chair").person + multichair = RoleFactory(group__type_id="rg", name_id="chair").person + RoleFactory(group__type_id="wg", person=multichair, name_id="chair") + ad = RoleFactory(group__type_id="area", name_id="ad").person + secretary = Role.objects.filter( + name_id="secr", group__acronym="secretariat" + ).first() + self.assertIsNotNone(secretary) + secretary = secretary.person + self.assertFalse( + has_role(rgchair.user, ["Secretariat", "Area Director", "WG Chair"]) + ) + url = urlreverse( + "ietf.doc.views_doc.document_main", kwargs={"name": draft.name} + ) + ask_about_ietf_link = urlreverse( + "ietf.doc.views_draft.ask_about_ietf_adoption_call", + kwargs={"name": draft.name}, + ) + non_ietf_adoption_link = urlreverse( + "ietf.doc.views_draft.adopt_draft", kwargs={"name": draft.name} + ) + for person in (None, nobody, rgchair, wgchair, multichair, ad, secretary): + if person is not None: + self.client.login( + username=person.user.username, + password=f"{person.user.username}+password", + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + has_ask_about_ietf_link = len(q(f'a[href="{ask_about_ietf_link}"]')) != 0 + has_non_ietf_adoption_link = ( + len(q(f'a[href="{non_ietf_adoption_link}"]')) != 0 + ) + ask_about_r = self.client.get(ask_about_ietf_link) + ask_about_link_return_code = ask_about_r.status_code + if person == rgchair: + self.assertFalse(has_ask_about_ietf_link) + self.assertTrue(has_non_ietf_adoption_link) + self.assertEqual(ask_about_link_return_code, 403) + elif person in (ad, nobody, None): + self.assertFalse(has_ask_about_ietf_link) + self.assertFalse(has_non_ietf_adoption_link) + self.assertEqual( + ask_about_link_return_code, 302 if person is None else 403 + ) + else: + self.assertTrue(has_ask_about_ietf_link) + self.assertFalse(has_non_ietf_adoption_link) + self.assertEqual(ask_about_link_return_code, 200) + self.client.logout() + + def test_ask_about_ietf_adoption_call(self): + # Basic permission tests above + doc = IndividualDraftFactory() + self.assertEqual(doc.docevent_set.count(), 1) + chair_role = RoleFactory(group__type_id="wg", name_id="chair") + chair = chair_role.person + group = chair_role.group + othergroup = GroupFactory(type_id="wg") + url = urlreverse( + "ietf.doc.views_draft.ask_about_ietf_adoption_call", + kwargs={"name": doc.name}, + ) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.post(url, {"group": othergroup.pk}) + self.assertEqual(r.status_code, 200) + r = self.client.post(url, {"group": group.pk}) + self.assertEqual(r.status_code, 302) + + def test_offer_wg_action_helpers(self): + def _assert_view_presents_buttons(testcase, response, expected): + q = PyQuery(response.content) + for id, expect in expected: + button = q(f"#{id}") + testcase.assertEqual( + len(button) != 0, + expect + ) + + # View rejects access + came_from_draft = WgDraftFactory(states=[("draft","rfc")]) + rfc = WgRfcFactory(group=came_from_draft.group) + came_from_draft.relateddocument_set.create(relationship_id="became_rfc",target=rfc) + rfc_chair = RoleFactory(name_id="chair", group=rfc.group).person + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=came_from_draft.name)) + login_testing_unauthorized(self, rfc_chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + self.client.logout() + rg_draft = RgDraftFactory() + rg_chair = RoleFactory(group=rg_draft.group, name_id="chair").person + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=rg_draft.name)) + login_testing_unauthorized(self, rg_chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code,404) + self.client.logout() + + # View offers access + draft = WgDraftFactory() + chair = RoleFactory(group=draft.group, name_id="chair").person + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=draft.name)) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code,200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", True), + ("id_pubreq_button", True), + ], + ) + draft.set_state(State.objects.get(type_id="draft-stream-ietf", slug="wg-cand")) + r = self.client.get(url) + self.assertEqual(r.status_code,200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", True), + ("id_wglc_button", False), + ("id_pubreq_button", False), + ], + ) + draft.set_state(State.objects.get(type_id="draft-stream-ietf", slug="wg-lc")) + StateDocEventFactory( + doc=draft, + state_type_id="draft-stream-ietf", + state=("draft-stream-ietf", "wg-lc"), + ) + self.assertEqual(draft.docevent_set.count(), 2) + r = self.client.get(url) + self.assertEqual(r.status_code,200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", False), + ("id_pubreq_button", True), + ], + ) + draft.set_state(State.objects.get(type_id="draft-stream-ietf",slug="chair-w")) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", True), + ("id_pubreq_button", True), + ], + ) + self.assertContains(response=r,text="Issue Another Working Group Last Call", status_code=200) + other_draft = WgDraftFactory() + self.client.logout() + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=other_draft.name)) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", True), + ("id_pubreq_button", True), + ], + ) + self.assertContains( + response=r, text="Issue Working Group Last Call", status_code=200 + ) + class BallotEmailAjaxTests(TestCase): def test_ajax_build_position_email(self): def _post_json(self, url, json_to_post): diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py index 8e9c0569e2..61e94b2231 100644 --- a/ietf/doc/urls.py +++ b/ietf/doc/urls.py @@ -125,6 +125,7 @@ url(r'^%(name)s/edit/info/$' % settings.URL_REGEXPS, views_draft.edit_info), url(r'^%(name)s/edit/requestresurrect/$' % settings.URL_REGEXPS, views_draft.request_resurrect), url(r'^%(name)s/edit/submit-to-iesg/$' % settings.URL_REGEXPS, views_draft.to_iesg), + url(r'^%(name)s/edit/issue-wg-lc/$' % settings.URL_REGEXPS, views_draft.issue_wg_lc), url(r'^%(name)s/edit/resurrect/$' % settings.URL_REGEXPS, views_draft.resurrect), url(r'^%(name)s/edit/addcomment/$' % settings.URL_REGEXPS, views_doc.add_comment), @@ -143,9 +144,13 @@ url(r'^%(name)s/edit/shepherdemail/$' % settings.URL_REGEXPS, views_draft.change_shepherd_email), url(r'^%(name)s/edit/shepherdwriteup/$' % settings.URL_REGEXPS, views_draft.edit_shepherd_writeup), url(r'^%(name)s/edit/requestpublication/$' % settings.URL_REGEXPS, views_draft.request_publication), + url(r'^%(name)s/edit/ask-about-ietf-adoption/$' % settings.URL_REGEXPS, views_draft.ask_about_ietf_adoption_call), url(r'^%(name)s/edit/adopt/$' % settings.URL_REGEXPS, views_draft.adopt_draft), + url(r'^%(name)s/edit/issue-wg-call-for-adoption/%(acronym)s/$' % settings.URL_REGEXPS, views_draft.issue_wg_call_for_adoption), + url(r'^%(name)s/edit/release/$' % settings.URL_REGEXPS, views_draft.release_draft), url(r'^%(name)s/edit/state/(?Pdraft-stream-[a-z]+)/$' % settings.URL_REGEXPS, views_draft.change_stream_state), + url(r'^%(name)s/edit/wg-action-helpers/$' % settings.URL_REGEXPS, views_draft.offer_wg_action_helpers), url(r'^%(name)s/edit/state/statement/$' % settings.URL_REGEXPS, views_statement.change_statement_state), url(r'^%(name)s/edit/clearballot/(?P[\w-]+)/$' % settings.URL_REGEXPS, views_ballot.clear_ballot), diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 5210317325..5564904504 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -515,13 +515,17 @@ def document_main(request, name, rev=None, document_html=False): # remaining actions actions = [] - if can_adopt_draft(request.user, doc) and not doc.get_state_slug() in ["rfc"] and not snapshot: + if can_adopt_draft(request.user, doc) and doc.get_state_slug() not in ["rfc"] and not snapshot: + target = urlreverse("ietf.doc.views_draft.adopt_draft", kwargs=dict(name=doc.name)) if doc.group and doc.group.acronym != 'none': # individual submission # already adopted in one group button_text = "Switch adoption" else: button_text = "Manage adoption" - actions.append((button_text, urlreverse('ietf.doc.views_draft.adopt_draft', kwargs=dict(name=doc.name)))) + # can_adopt_draft currently returns False for Area Directors + if has_role(request.user, ["Secretariat", "WG Chair"]): + target = urlreverse("ietf.doc.views_draft.ask_about_ietf_adoption_call", kwargs=dict(name=doc.name)) + actions.append((button_text, target)) if can_unadopt_draft(request.user, doc) and not doc.get_state_slug() in ["rfc"] and not snapshot: if doc.get_state_slug('draft-iesg') == 'idexists': diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py index 16d04ee66a..c5faf1140b 100644 --- a/ietf/doc/views_draft.py +++ b/ietf/doc/views_draft.py @@ -28,12 +28,12 @@ IanaExpertDocEvent, IESG_SUBSTATE_TAGS) from ietf.doc.mails import ( email_pulled_from_rfc_queue, email_resurrect_requested, email_resurrection_completed, email_state_changed, email_stream_changed, - email_wg_call_for_adoption_issued, email_wg_last_call_issued, email_stream_state_changed, email_stream_tags_changed, extra_automation_headers, generate_publication_request, email_adopted, email_intended_status_changed, email_iesg_processing_document, email_ad_approved_doc, email_iana_expert_review_state_changed ) from ietf.doc.storage_utils import retrieve_bytes, store_bytes +from ietf.doc.templatetags.ietf_filters import is_doc_ietf_adoptable from ietf.doc.utils import ( add_state_change_event, can_adopt_draft, can_unadopt_draft, get_tags_for_stream_id, nice_consensus, update_action_holders, update_reminder, update_telechat, make_notify_changed_event, get_initial_notify, @@ -51,12 +51,12 @@ from ietf.name.models import IntendedStdLevelName, DocTagName, StreamName from ietf.person.fields import SearchableEmailField from ietf.person.models import Person, Email -from ietf.utils.mail import send_mail, send_mail_message, on_behalf_of +from ietf.utils.mail import send_mail, send_mail_message, on_behalf_of, send_mail_text from ietf.utils.textupload import get_cleaned_text_file_content from ietf.utils import log -from ietf.utils.fields import ModelMultipleChoiceField +from ietf.utils.fields import DatepickerDateField, ModelMultipleChoiceField, MultiEmailField from ietf.utils.response import permission_denied -from ietf.utils.timezone import datetime_today, DEADLINE_TZINFO +from ietf.utils.timezone import date_today, datetime_from_date, datetime_today, DEADLINE_TZINFO class ChangeStateForm(forms.Form): @@ -1564,7 +1564,7 @@ def adopt_draft(request, name): events.append(e) due_date = None - if form.cleaned_data["weeks"] != None: + if form.cleaned_data["weeks"] is not None: due_date = datetime_today(DEADLINE_TZINFO) + datetime.timedelta(weeks=form.cleaned_data["weeks"]) update_reminder(doc, "stream-s", e, due_date) @@ -1573,11 +1573,6 @@ def adopt_draft(request, name): # setting states that are _not_ the adopted state. email_adopted(request, doc, prev_state, new_state, by, comment) - # Currently only the IETF stream uses the c-adopt state - guard against other - # streams starting to use it asthe IPR rules for those streams will be different. - if doc.stream_id == "ietf" and new_state.slug == "c-adopt": - email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=form.cleaned_data["weeks"]) - # comment if comment: e = DocEvent(type="added_comment", doc=doc, rev=doc.rev, by=by) @@ -1689,11 +1684,14 @@ def __init__(self, *args, **kwargs): f.queryset = f.queryset.exclude(pk__in=unused_states) f.label = state_type.label if self.stream.slug == 'ietf': + help_text_items = [] if self.can_set_sub_pub: - f.help_text = "Only select 'Submitted to IESG for Publication' to correct errors. Use the document's main page to request publication." + help_text_items.append("Only select 'Submitted to IESG for Publication' to correct errors. This is not how to submit a document to the IESG.") else: f.queryset = f.queryset.exclude(slug='sub-pub') - f.help_text = "You may not set the 'Submitted to IESG for Publication' using this form - Use the document's main page to request publication." + help_text_items.append("You may not set the 'Submitted to IESG for Publication' using this form - Use the button above or the document's main page to request publication.") + help_text_items.append("Only use this form in unusual circumstances when issuing call for adoption or working group last call.") + f.help_text = " ".join(help_text_items) f = self.fields['tags'] f.queryset = f.queryset.filter(slug__in=get_tags_for_stream_id(doc.stream_id)) @@ -1704,7 +1702,7 @@ def __init__(self, *args, **kwargs): def clean_new_state(self): new_state = self.cleaned_data.get('new_state') if new_state.slug=='sub-pub' and not self.can_set_sub_pub: - raise forms.ValidationError('You may not set the %s state using this form. Use the "Submit to IESG for publication" button on the document\'s main page instead. If that button does not appear, the document may already have IESG state. Ask your Area Director or the Secretariat for help.'%new_state.name) + raise forms.ValidationError('You may not set the %s state using this form. Use the "Submit to IESG for Publication" button on the document\'s main page instead. If that button does not appear, the document may already have IESG state. Ask your Area Director or the Secretariat for help.'%new_state.name) return new_state @@ -1730,6 +1728,19 @@ def next_states_for_stream_state(doc, state_type, current_state): return next_states +@login_required +def offer_wg_action_helpers(request, name): + doc = get_object_or_404(Document, type="draft", name=name) + if doc.stream is None or doc.stream_id != "ietf" or doc.became_rfc() is not None: + raise Http404 + + if not is_authorized_in_doc_stream(request.user, doc): + permission_denied(request, "You don't have permission to access this page.") + + return render(request, "doc/draft/wg_action_helpers.html", + {"doc": doc, + }) + @login_required def change_stream_state(request, name, state_type): doc = get_object_or_404(Document, type="draft", name=name) @@ -1744,10 +1755,17 @@ def change_stream_state(request, name, state_type): prev_state = doc.get_state(state_type.slug) next_states = next_states_for_stream_state(doc, state_type, prev_state) + # These tell the form to allow directly setting the state to fix up errors. can_set_sub_pub = has_role(request.user,('Secretariat','Area Director')) or (prev_state and prev_state.slug=='sub-pub') if request.method == 'POST': - form = ChangeStreamStateForm(request.POST, doc=doc, state_type=state_type,can_set_sub_pub=can_set_sub_pub,stream=doc.stream) + form = ChangeStreamStateForm( + request.POST, + doc=doc, + state_type=state_type, + can_set_sub_pub=can_set_sub_pub, + stream=doc.stream, + ) if form.is_valid(): by = request.user.person events = [] @@ -1768,14 +1786,7 @@ def change_stream_state(request, name, state_type): update_reminder(doc, "stream-s", e, due_date) email_stream_state_changed(request, doc, prev_state, new_state, by, comment) - - if doc.stream_id == "ietf": - if new_state.slug == "c-adopt": - email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=form.cleaned_data["weeks"]) - if new_state.slug == "wg-lc": - email_wg_last_call_issued(request, doc, wglc_duration_weeks=form.cleaned_data["weeks"]) - # tags existing_tags = set(doc.tags.all()) new_tags = set(form.cleaned_data["tags"]) @@ -1811,8 +1822,15 @@ def change_stream_state(request, name, state_type): else: form.add_error(None, "No change in state or tags found, and no comment provided -- nothing to do.") else: - form = ChangeStreamStateForm(initial=dict(new_state=prev_state.pk if prev_state else None, tags= doc.tags.all()), - doc=doc, state_type=state_type, can_set_sub_pub = can_set_sub_pub,stream = doc.stream) + form = ChangeStreamStateForm( + initial=dict( + new_state=prev_state.pk if prev_state else None, tags=doc.tags.all() + ), + doc=doc, + state_type=state_type, + can_set_sub_pub=can_set_sub_pub, + stream=doc.stream, + ) milestones = doc.groupmilestone_set.all() @@ -1857,3 +1875,325 @@ def set_intended_status_level(request, doc, new_level, old_level, comment): msg = "\n".join(e.desc for e in events) email_intended_status_changed(request, doc, msg) + +class IssueWorkingGroupLastCallForm(forms.Form): + end_date = DatepickerDateField( + required=True, + date_format="yyyy-mm-dd", + picker_settings={ + "autoclose": "1", + }, + help_text="The date the Last Call closes. If you change this, review the subject and body carefully to ensure the change is captured correctly.", + ) + + to = MultiEmailField( + required=True, + help_text="Comma separated list of address to use in the To: header", + ) + cc = MultiEmailField( + required=False, help_text="Comma separated list of addresses to copy" + ) + subject = forms.CharField( + required=True, + help_text="Subject for Last Call message. If you change the date here, be sure to make a matching change in the body.", + ) + body = forms.CharField( + widget=forms.Textarea, required=True, help_text="Body for Last Call message" + ) + + def clean_end_date(self): + end_date = self.cleaned_data["end_date"] + if end_date <= date_today(DEADLINE_TZINFO): + raise forms.ValidationError("End date must be later than today") + return end_date + + def clean(self): + cleaned_data = super().clean() + end_date = cleaned_data.get("end_date") + if end_date is not None: + body = cleaned_data.get("body") + subject = cleaned_data.get("subject") + if end_date.isoformat() not in body: + self.add_error( + "body", + forms.ValidationError( + f"Last call end date ({end_date.isoformat()}) not found in body" + ), + ) + if end_date.isoformat() not in subject: + self.add_error( + "subject", + forms.ValidationError( + f"Last call end date ({end_date.isoformat()}) not found in subject" + ), + ) + return cleaned_data + + +@login_required +def issue_wg_lc(request, name): + doc = get_object_or_404(Document, name=name) + + if doc.stream_id != "ietf": + raise Http404 + if doc.type_id != "draft" or doc.group.type_id != "wg": + raise Http404 + if doc.get_state_slug("draft-stream-ietf") == "wg-lc": + raise Http404 + if doc.get_state_slug("draft") == "rfc": + raise Http404 + + if not is_authorized_in_doc_stream(request.user, doc): + permission_denied(request, "You don't have permission to access this page.") + + if request.method == "POST": + form = IssueWorkingGroupLastCallForm(request.POST) + if form.is_valid(): + # Intentionally not changing tags or adding a comment + # those things can be done with other workflows + by = request.user.person + prev_state = doc.get_state("draft-stream-ietf") + events = [] + wglc_state = State.objects.get(type="draft-stream-ietf", slug="wg-lc") + doc.set_state(wglc_state) + e = add_state_change_event(doc, by, prev_state, wglc_state) + events.append(e) + end_date = form.cleaned_data["end_date"] + update_reminder( + doc, "stream-s", e, datetime_from_date(end_date, DEADLINE_TZINFO) + ) + doc.save_with_history(events) + email_stream_state_changed(request, doc, prev_state, wglc_state, by) + send_mail_text( + request, + to = form.cleaned_data["to"], + frm = request.user.person.formatted_email(), + subject = form.cleaned_data["subject"], + txt = form.cleaned_data["body"], + cc = form.cleaned_data["cc"], + ) + return redirect("ietf.doc.views_doc.document_main", name=doc.name) + else: + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=14) + subject = f"WG Last Call: {doc.name}-{doc.rev} (Ends {end_date})" + body = render_to_string( + "doc/mail/wg_last_call_issued.txt", + dict( + doc=doc, + end_date=end_date, + wg_list=doc.group.list_email, + settings=settings, + ), + ) + (to, cc) = gather_address_lists("doc_wg_last_call_issued", doc=doc) + + form = IssueWorkingGroupLastCallForm( + initial=dict( + end_date=end_date, + to=", ".join(to), + cc=", ".join(cc), + subject=subject, + body=body, + ) + ) + + return render( + request, + "doc/draft/issue_working_group_last_call.html", + dict( + doc=doc, + form=form, + ), + ) + +class IssueCallForAdoptionForm(forms.Form): + end_date = DatepickerDateField( + required=True, + date_format="yyyy-mm-dd", + picker_settings={ + "autoclose": "1", + }, + help_text="The date the Call for Adoption closes. If you change this, review the subject and body carefully to ensure the change is captured correctly.", + ) + + to = MultiEmailField( + required=True, + help_text="Comma separated list of address to use in the To: header", + ) + cc = MultiEmailField( + required=False, help_text="Comma separated list of addresses to copy" + ) + subject = forms.CharField( + required=True, + help_text="Subject for Call for Adoption message. If you change the date here, be sure to make a matching change in the body.", + ) + body = forms.CharField( + widget=forms.Textarea, required=True, help_text="Body for Call for Adoption message" + ) + + def clean_end_date(self): + end_date = self.cleaned_data["end_date"] + if end_date <= date_today(DEADLINE_TZINFO): + raise forms.ValidationError("End date must be later than today") + return end_date + + def clean(self): + cleaned_data = super().clean() + end_date = cleaned_data.get("end_date") + if end_date is not None: + body = cleaned_data.get("body") + subject = cleaned_data.get("subject") + if end_date.isoformat() not in body: + self.add_error( + "body", + forms.ValidationError( + f"Call for adoption end date ({end_date.isoformat()}) not found in body" + ), + ) + if end_date.isoformat() not in subject: + self.add_error( + "subject", + forms.ValidationError( + f"Call for adoption end date ({end_date.isoformat()}) not found in subject" + ), + ) + return cleaned_data + +@login_required +def issue_wg_call_for_adoption(request, name, acronym): + doc = get_object_or_404(Document, name=name) + group = Group.objects.filter(acronym=acronym, type_id="wg").first() + reject = False + if group is None or doc.type_id != "draft" or not is_doc_ietf_adoptable(doc): + reject = True + if doc.stream is None: + if not can_adopt_draft(request.user, doc): + reject = True + elif doc.stream_id != "ietf": + reject = True + else: # doc.stream_id == "ietf" + if not is_authorized_in_doc_stream(request.user, doc): + reject = True + if reject: + raise permission_denied(request, f"You can't issue a {acronym} wg call for adoption for this document.") + + if request.method == "POST": + form = IssueCallForAdoptionForm(request.POST) + if form.is_valid(): + # Intentionally not changing tags or adding a comment + # those things can be done with other workflows + by = request.user.person + + events = [] + if doc.stream_id != "ietf": + stream = StreamName.objects.get(slug="ietf") + doc.stream = stream + e = DocEvent(type="changed_stream", doc=doc, rev=doc.rev, by=by) + e.desc = f"Changed stream to {stream.name}" # Propogates embedding html in DocEvent.desc for consistency + e.save() + events.append(e) + if doc.group != group: + doc.group = group + e = DocEvent(type="changed_group", doc=doc, rev=doc.rev, by=by) + e.desc = f"Changed group to {group.name} ({group.acronym.upper()})" # Even if it makes the cats cry + e.save() + events.append(e) + prev_state = doc.get_state("draft-stream-ietf") + c_adopt_state = State.objects.get(type="draft-stream-ietf", slug="c-adopt") + doc.set_state(c_adopt_state) + e = add_state_change_event(doc, by, prev_state, c_adopt_state) + events.append(e) + end_date = form.cleaned_data["end_date"] + update_reminder( + doc, "stream-s", e, datetime_from_date(end_date, DEADLINE_TZINFO) + ) + doc.save_with_history(events) + email_stream_state_changed(request, doc, prev_state, c_adopt_state, by) + send_mail_text( + request, + to = form.cleaned_data["to"], + frm = request.user.person.formatted_email(), + subject = form.cleaned_data["subject"], + txt = form.cleaned_data["body"], + cc = form.cleaned_data["cc"], + ) + return redirect("ietf.doc.views_doc.document_main", name=doc.name) + else: + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=14) + subject = f"Call for adoption: {doc.name}-{doc.rev} (Ends {end_date})" + body = render_to_string( + "doc/mail/wg_call_for_adoption_issued.txt", + dict( + doc=doc, + group=group, + end_date=end_date, + wg_list=doc.group.list_email, + settings=settings, + ), + ) + (to, cc) = gather_address_lists("doc_wg_call_for_adoption_issued", doc=doc) + if doc.group.acronym == "none": + to.insert(0, f"{group.acronym}-chairs@ietf.org") + to.insert(0, group.list_email) + form = IssueCallForAdoptionForm( + initial=dict( + end_date=end_date, + to=", ".join(to), + cc=", ".join(cc), + subject=subject, + body=body, + ) + ) + + return render( + request, + "doc/draft/issue_working_group_call_for_adoption.html", + dict( + doc=doc, + form=form, + ), + ) + +class GroupModelChoiceField(forms.ModelChoiceField): + def label_from_instance(self, obj): + return f"{obj.acronym} - {obj.name}" + + +class WgForm(forms.Form): + group = GroupModelChoiceField( + queryset=Group.objects.filter(type_id="wg", state="active") + .order_by("acronym") + .distinct(), + required=True, + empty_label="Select IETF Working Group", + ) + + def __init__(self, *args, **kwargs): + user = kwargs.pop("user") + super(WgForm, self).__init__(*args, **kwargs) + if not has_role(user, ["Secretariat", "Area Director"]): + self.fields["group"].queryset = self.fields["group"].queryset.filter( + role__name_id="chair", role__person=user.person + ) + + +@role_required("Secretariat", "WG Chair") +def ask_about_ietf_adoption_call(request, name): + doc = get_object_or_404(Document, name=name) + if doc.stream is not None or doc.group.acronym != "none": + raise Http404 + if request.method == "POST": + form = WgForm(request.POST, user=request.user) + if form.is_valid(): + group = form.cleaned_data["group"] + return redirect(issue_wg_call_for_adoption, name=doc.name, acronym=group.acronym) + else: + form = WgForm(initial={"group": None}, user=request.user) + return render( + request, + "doc/draft/ask_about_ietf_adoption.html", + dict( + doc=doc, + form=form, + ), + ) diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py index e2893a90f7..1f634278be 100644 --- a/ietf/ietfauth/utils.py +++ b/ietf/ietfauth/utils.py @@ -211,9 +211,9 @@ def role_required(*role_names): # specific permissions + def is_authorized_in_doc_stream(user, doc): - """Return whether user is authorized to perform stream duties on - document.""" + """Is user authorized to perform stream duties on doc?""" if has_role(user, ["Secretariat"]): return True diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index 6414538283..eab1d779fb 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -63,7 +63,12 @@ {% if doc.stream and can_edit_stream_info and doc.stream.slug != "legacy" and not snapshot %} + {% if doc|is_doc_ietf_adoptable or doc|can_issue_ietf_wg_lc or doc|can_submit_to_iesg %} + href="{% url 'ietf.doc.views_draft.offer_wg_action_helpers' name=doc.name %}" + {% else %} + href="{% url 'ietf.doc.views_draft.change_stream_state' name=doc.name state_type=stream_state_type_slug %}" + {% endif %} + > Edit {% endif %} diff --git a/ietf/templates/doc/draft/ask_about_ietf_adoption.html b/ietf/templates/doc/draft/ask_about_ietf_adoption.html new file mode 100644 index 0000000000..d19e4572b7 --- /dev/null +++ b/ietf/templates/doc/draft/ask_about_ietf_adoption.html @@ -0,0 +1,22 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} +{% load django_bootstrap5 ietf_filters origin %} +{% block title %}Manage Adoption of {{ doc }}{% endblock %} +{% block content %} + {% origin %} +

        + Manage Adoption +
        + {{ doc }} +

        +
        + Do you wish to issue an IETF Working Group call for adoption to one of these working groups? +
        +
        + {% csrf_token %} + {% bootstrap_form form %} + + No, I wish to manage adoption directly, perhaps with non-IETF-stream groups + Back +
        +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/draft/change_stream_state.html b/ietf/templates/doc/draft/change_stream_state.html index 0b13e02fdf..7f3132b3c8 100644 --- a/ietf/templates/doc/draft/change_stream_state.html +++ b/ietf/templates/doc/draft/change_stream_state.html @@ -1,7 +1,6 @@ {% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin %} -{% load django_bootstrap5 %} +{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} +{% load django_bootstrap5 ietf_filters origin %} {% block title %}Change {{ state_type.label }} for {{ doc }}{% endblock %} {% block content %} {% origin %} @@ -14,12 +13,10 @@

        Help on states

        - Move document to {{ next_states|pluralize:"to one of" }} the recommended next state{{ next_states|pluralize }}: + Move document to {{ next_states|pluralize:"one of" }} the recommended next state{{ next_states|pluralize }}:

        {% for state in next_states %} - {% if state.slug == 'sub-pub' %} - {{ state.name }} - {% else %} + {% if state.slug != 'sub-pub' and state.slug != "wg-lc" %} {% endif %} {% endfor %} @@ -28,7 +25,13 @@

        {% csrf_token %} {% bootstrap_form form %} - Back + Back {% endblock %} {% block js %} diff --git a/ietf/templates/doc/draft/issue_working_group_call_for_adoption.html b/ietf/templates/doc/draft/issue_working_group_call_for_adoption.html new file mode 100644 index 0000000000..61094b053a --- /dev/null +++ b/ietf/templates/doc/draft/issue_working_group_call_for_adoption.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load origin django_bootstrap5 static %} +={% block title %}Issue Working Group Call for Adoption of {{ doc }}{% endblock %} +{% block pagehead %} + +{% endblock %} +{% block content %} + {% origin %} +

        + Issue Working Group Call for Adoption +
        + {{ doc }} +

        + {% if form.errors %} +

        + Please correct the following: +

        + {% endif %} + {% bootstrap_form_errors form %} +
        + {% csrf_token %} + {% bootstrap_form form %} + + + Back +
        +{% endblock %} +{% block js %} + + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/draft/issue_working_group_last_call.html b/ietf/templates/doc/draft/issue_working_group_last_call.html new file mode 100644 index 0000000000..d6f35a0e82 --- /dev/null +++ b/ietf/templates/doc/draft/issue_working_group_last_call.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load origin django_bootstrap5 static %} +={% block title %}Issue Working Group Last Call for {{ doc }}{% endblock %} +{% block pagehead %} + +{% endblock %} +{% block content %} + {% origin %} +

        + Issue Working Group Last Call +
        + {{ doc }} +

        + {% if form.errors %} +

        + Please correct the following: +

        + {% endif %} + {% bootstrap_form_errors form %} +
        + {% csrf_token %} + {% bootstrap_form form %} + + + Back +
        +{% endblock %} +{% block js %} + + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/draft/wg_action_helpers.html b/ietf/templates/doc/draft/wg_action_helpers.html new file mode 100644 index 0000000000..d21f3c0926 --- /dev/null +++ b/ietf/templates/doc/draft/wg_action_helpers.html @@ -0,0 +1,25 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} +{% load django_bootstrap5 ietf_filters origin %} +{% block title %}Change IETF WG state for {{ doc }}{% endblock %} +{% block content %} + {% origin %} +

        + Change IETF WG state +
        + {{ doc }} +

        +
        + {% if doc|is_doc_ietf_adoptable %} + Issue WG Call for Adoption + {% endif %} + {% if doc|can_issue_ietf_wg_lc %} + Issue{% if doc|has_had_ietf_wg_lc %} Another{% endif %} Working Group Last Call + {% endif %} + {% if doc|can_submit_to_iesg %} + Submit to IESG for Publication + {% endif %} + Set any WG state directly + Back +
        +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt b/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt index c4a2401bc2..15ace9495b 100644 --- a/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt +++ b/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt @@ -1,15 +1,13 @@ -{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %} -Subject: {{ subject }} +{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %}This message starts a {{group.acronym}} WG Call for Adoption of: +{{ doc.name }}-{{ doc.rev }} -This message starts a {{ cfa_duration_weeks }}-week Call for Adoption for this document. + +This Working Group Call for Adoption ends on {{ end_date }} Abstract: {{ doc.abstract }} -File can be retrieved from: -{{ url }} - -Please reply to this message keeping {{ wg_list }} in copy by indicating whether you support or not the adoption of this draft as a WG document. Comments to motivate your preference are highly appreciated. +Please reply to this message and indicate whether or not you support adoption of this Internet-Draft by the {{group.acronym}} WG. Comments to explain your preference are greatly appreciated. Please reply to all recipients of this message and include this message in your response. Authors, and WG participants in general, are reminded of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [2]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3]. @@ -17,5 +15,17 @@ Thank you. [1] https://datatracker.ietf.org/doc/bcp78/ [2] https://datatracker.ietf.org/doc/bcp79/ [3] https://datatracker.ietf.org/doc/rfc6701/ + +The IETF datatracker status page for this Internet-Draft is: +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name=doc.name %} +{% if doc.submission.xml_version == "3" %} +There is also an HTML version available at: +{{ settings.IETF_ID_ARCHIVE_URL }}{{ doc.name }}-{{ doc.rev }}.html{% else %} +There is also an HTMLized version available at: +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_html' name=doc.name rev=doc.rev %}{% endif %} +{% if doc.rev != "00" %} +A diff from the previous version is available at: +{{settings.RFCDIFF_BASE_URL}}?url2={{ doc.name }}-{{ doc.rev }} +{% endif %} {% endfilter %} {% endautoescape %} diff --git a/ietf/templates/doc/mail/wg_last_call_issued.txt b/ietf/templates/doc/mail/wg_last_call_issued.txt index 35b1e149d7..114f8bc5e2 100644 --- a/ietf/templates/doc/mail/wg_last_call_issued.txt +++ b/ietf/templates/doc/mail/wg_last_call_issued.txt @@ -1,7 +1,7 @@ -{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %} -Subject: {{ subject }} +{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %}This message starts a WG Last Call for: +{{ doc.name }}-{{ doc.rev }} -This message starts a {{ wglc_duration_weeks }}-week WG Last Call for this document. +This Working Group Last Call ends on {{ end_date }} Abstract: {{ doc.abstract }} @@ -9,14 +9,26 @@ Abstract: File can be retrieved from: {{ url }} -Please review and indicate your support or objection to proceed with the publication of this document by replying to this email keeping {{ wg_list }} in copy. Objections should be motivated and suggestions to resolve them are highly appreciated. +Please review and indicate your support or objection to proceed with the publication of this document by replying to this email keeping {{ wg_list }} in copy. Objections should be explained and suggestions to resolve them are highly appreciated. -Authors, and WG participants in general, are reminded again of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [1]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3]. +Authors, and WG participants in general, are reminded of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [1]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3]. Thank you. [1] https://datatracker.ietf.org/doc/bcp78/ [2] https://datatracker.ietf.org/doc/bcp79/ [3] https://datatracker.ietf.org/doc/rfc6701/ + +The IETF datatracker status page for this Internet-Draft is: +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name=doc.name %} +{% if doc.submission.xml_version == "3" %} +There is also an HTML version available at: +{{ settings.IETF_ID_ARCHIVE_URL }}{{ doc.name }}-{{ doc.rev }}.html{% else %} +There is also an HTMLized version available at: +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_html' name=doc.name rev=doc.rev %}{% endif %} +{% if doc.rev != "00" %} +A diff from the previous version is available at: +{{settings.RFCDIFF_BASE_URL}}?url2={{ doc.name }}-{{ doc.rev }} +{% endif %} {% endfilter %} {% endautoescape %} From fb50ac0d90f7a8d5b5ebe7f88ce08be2e9ea2acf Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 16 Dec 2025 11:58:05 -0400 Subject: [PATCH 029/158] chore: optional opentelemetry (#10112) * chore: optional opentelemetry * chore: add comment --- dev/build/gunicorn.conf.py | 45 +++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py index c54b24a054..9af4478685 100644 --- a/dev/build/gunicorn.conf.py +++ b/dev/build/gunicorn.conf.py @@ -135,21 +135,30 @@ def post_request(worker, req, environ, resp): def post_fork(server, worker): server.log.info("Worker spawned (pid: %s)", worker.pid) - resource = Resource.create(attributes={ - "service.name": "datatracker", - "service.version": ietf.__version__, - "service.instance.id": worker.pid, - "service.namespace": "datatracker", - "deployment.environment.name": os.environ.get("DATATRACKER_SERVICE_ENV", "dev") - }) - - trace.set_tracer_provider(TracerProvider(resource=resource)) - otlp_exporter = OTLPSpanExporter(endpoint="https://heimdall-otlp.ietf.org/v1/traces") - - trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter)) - - # Instrumentations - DjangoInstrumentor().instrument() - Psycopg2Instrumentor().instrument() - PymemcacheInstrumentor().instrument() - RequestsInstrumentor().instrument() + # Setting DATATRACKER_OPENTELEMETRY_ENABLE=all in the environment will enable all + # opentelemetry instrumentations. Individual instrumentations can be selected by + # using a space-separated list. See the code below for available instrumentations. + telemetry_env = os.environ.get("DATATRACKER_OPENTELEMETRY_ENABLE", "").strip() + if telemetry_env != "": + enabled_telemetry = [tok.strip().lower() for tok in telemetry_env.split()] + resource = Resource.create(attributes={ + "service.name": "datatracker", + "service.version": ietf.__version__, + "service.instance.id": worker.pid, + "service.namespace": "datatracker", + "deployment.environment.name": os.environ.get("DATATRACKER_SERVICE_ENV", "dev") + }) + trace.set_tracer_provider(TracerProvider(resource=resource)) + otlp_exporter = OTLPSpanExporter(endpoint="https://heimdall-otlp.ietf.org/v1/traces") + + trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter)) + + # Instrumentations + if "all" in enabled_telemetry or "django" in enabled_telemetry: + DjangoInstrumentor().instrument() + if "all" in enabled_telemetry or "psycopg2" in enabled_telemetry: + Psycopg2Instrumentor().instrument() + if "all" in enabled_telemetry or "pymemcache" in enabled_telemetry: + PymemcacheInstrumentor().instrument() + if "all" in enabled_telemetry or "requests" in enabled_telemetry: + RequestsInstrumentor().instrument() From 58430dbdfdf306c107fe62cae97e23050688e56a Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 16 Dec 2025 13:44:41 -0400 Subject: [PATCH 030/158] refactor: no delete from inline admins (#10113) * refactor: no delete from TabularInlines * refactor: no delete from StackedInlines --- ietf/doc/admin.py | 11 +++++++---- ietf/group/admin.py | 7 ++++--- ietf/ipr/admin.py | 5 +++-- ietf/liaisons/admin.py | 5 +++-- ietf/meeting/admin.py | 9 +++++---- ietf/name/admin.py | 3 ++- ietf/person/admin.py | 5 +++-- ietf/utils/admin.py | 13 +++++++++++++ 8 files changed, 40 insertions(+), 18 deletions(-) diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index 745536f9a1..920e389a9a 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -15,6 +15,7 @@ ReviewAssignmentDocEvent, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject ) +from ietf.utils.admin import SaferTabularInline from ietf.utils.validators import validate_external_resource_value class StateTypeAdmin(admin.ModelAdmin): @@ -28,17 +29,19 @@ class StateAdmin(admin.ModelAdmin): filter_horizontal = ["next_states"] admin.site.register(State, StateAdmin) -class DocAuthorInline(admin.TabularInline): +class DocAuthorInline(SaferTabularInline): model = DocumentAuthor raw_id_fields = ['person', 'email'] extra = 1 + can_delete = False + show_change_link = True -class DocActionHolderInline(admin.TabularInline): +class DocActionHolderInline(SaferTabularInline): model = DocumentActionHolder raw_id_fields = ['person'] extra = 1 -class RelatedDocumentInline(admin.TabularInline): +class RelatedDocumentInline(SaferTabularInline): model = RelatedDocument fk_name= 'source' def this(self, instance): @@ -48,7 +51,7 @@ def this(self, instance): raw_id_fields = ['target'] extra = 1 -class AdditionalUrlInLine(admin.TabularInline): +class AdditionalUrlInLine(SaferTabularInline): model = DocumentURL fields = ['tag','desc','url',] extra = 1 diff --git a/ietf/group/admin.py b/ietf/group/admin.py index fedec49d85..685c10aeea 100644 --- a/ietf/group/admin.py +++ b/ietf/group/admin.py @@ -26,14 +26,15 @@ MilestoneGroupEvent, GroupExtResource, Appeal, AppealArtifact ) from ietf.name.models import GroupTypeName -from ietf.utils.validators import validate_external_resource_value +from ietf.utils.admin import SaferTabularInline from ietf.utils.response import permission_denied +from ietf.utils.validators import validate_external_resource_value -class RoleInline(admin.TabularInline): +class RoleInline(SaferTabularInline): model = Role raw_id_fields = ["person", "email"] -class GroupURLInline(admin.TabularInline): +class GroupURLInline(SaferTabularInline): model = GroupURL class GroupForm(forms.ModelForm): diff --git a/ietf/ipr/admin.py b/ietf/ipr/admin.py index 1a8a908dcd..d6a320203b 100644 --- a/ietf/ipr/admin.py +++ b/ietf/ipr/admin.py @@ -17,6 +17,7 @@ NonDocSpecificIprDisclosure, LegacyMigrationIprEvent, ) +from ietf.utils.admin import SaferTabularInline # ------------------------------------------------------ # ModelAdmins @@ -29,13 +30,13 @@ class Meta: 'sections':forms.TextInput, } -class IprDocRelInline(admin.TabularInline): +class IprDocRelInline(SaferTabularInline): model = IprDocRel form = IprDocRelAdminForm raw_id_fields = ['document'] extra = 1 -class RelatedIprInline(admin.TabularInline): +class RelatedIprInline(SaferTabularInline): model = RelatedIpr raw_id_fields = ['target'] fk_name = 'source' diff --git a/ietf/liaisons/admin.py b/ietf/liaisons/admin.py index 21515ed1a3..d873cce536 100644 --- a/ietf/liaisons/admin.py +++ b/ietf/liaisons/admin.py @@ -7,15 +7,16 @@ from ietf.liaisons.models import ( LiaisonStatement, LiaisonStatementEvent, RelatedLiaisonStatement, LiaisonStatementAttachment ) +from ietf.utils.admin import SaferTabularInline -class RelatedLiaisonStatementInline(admin.TabularInline): +class RelatedLiaisonStatementInline(SaferTabularInline): model = RelatedLiaisonStatement fk_name = 'source' raw_id_fields = ['target'] extra = 1 -class LiaisonStatementAttachmentInline(admin.TabularInline): +class LiaisonStatementAttachmentInline(SaferTabularInline): model = LiaisonStatementAttachment raw_id_fields = ['document'] extra = 1 diff --git a/ietf/meeting/admin.py b/ietf/meeting/admin.py index d886a9a4b6..03abf5c029 100644 --- a/ietf/meeting/admin.py +++ b/ietf/meeting/admin.py @@ -10,6 +10,7 @@ SessionPresentation, ImportantDate, SlideSubmission, SchedulingEvent, BusinessConstraint, ProceedingsMaterial, MeetingHost, Registration, RegistrationTicket, AttendanceTypeName) +from ietf.utils.admin import SaferTabularInline class UrlResourceAdmin(admin.ModelAdmin): @@ -18,7 +19,7 @@ class UrlResourceAdmin(admin.ModelAdmin): raw_id_fields = ['room', ] admin.site.register(UrlResource, UrlResourceAdmin) -class UrlResourceInline(admin.TabularInline): +class UrlResourceInline(SaferTabularInline): model = UrlResource class RoomAdmin(admin.ModelAdmin): @@ -28,7 +29,7 @@ class RoomAdmin(admin.ModelAdmin): admin.site.register(Room, RoomAdmin) -class RoomInline(admin.TabularInline): +class RoomInline(SaferTabularInline): model = Room class MeetingAdmin(admin.ModelAdmin): @@ -93,7 +94,7 @@ def name_lower(self, instance): admin.site.register(Constraint, ConstraintAdmin) -class SchedulingEventInline(admin.TabularInline): +class SchedulingEventInline(SaferTabularInline): model = SchedulingEvent raw_id_fields = ["by"] @@ -244,7 +245,7 @@ def queryset(self, request, queryset): return queryset.filter(tickets__attendance_type__slug=self.value()).distinct() return queryset -class RegistrationTicketInline(admin.TabularInline): +class RegistrationTicketInline(SaferTabularInline): model = RegistrationTicket class RegistrationAdmin(admin.ModelAdmin): diff --git a/ietf/name/admin.py b/ietf/name/admin.py index 4336e0569c..b89d6d141c 100644 --- a/ietf/name/admin.py +++ b/ietf/name/admin.py @@ -57,6 +57,7 @@ from ietf.stats.models import CountryAlias +from ietf.utils.admin import SaferTabularInline class NameAdmin(admin.ModelAdmin): @@ -86,7 +87,7 @@ class GroupTypeNameAdmin(NameAdmin): admin.site.register(GroupTypeName, GroupTypeNameAdmin) -class CountryAliasInline(admin.TabularInline): +class CountryAliasInline(SaferTabularInline): model = CountryAlias extra = 1 diff --git a/ietf/person/admin.py b/ietf/person/admin.py index cd8ca2abf1..f46edcf8ae 100644 --- a/ietf/person/admin.py +++ b/ietf/person/admin.py @@ -7,6 +7,7 @@ from ietf.person.models import Email, Alias, Person, PersonalApiKey, PersonEvent, PersonApiKeyEvent, PersonExtResource from ietf.person.name import name_parts +from ietf.utils.admin import SaferStackedInline, SaferTabularInline from ietf.utils.validators import validate_external_resource_value @@ -16,7 +17,7 @@ class EmailAdmin(simple_history.admin.SimpleHistoryAdmin): search_fields = ["address", "person__name", ] admin.site.register(Email, EmailAdmin) -class EmailInline(admin.TabularInline): +class EmailInline(SaferTabularInline): model = Email class AliasAdmin(admin.ModelAdmin): @@ -25,7 +26,7 @@ class AliasAdmin(admin.ModelAdmin): raw_id_fields = ["person"] admin.site.register(Alias, AliasAdmin) -class AliasInline(admin.StackedInline): +class AliasInline(SaferStackedInline): model = Alias class PersonAdmin(simple_history.admin.SimpleHistoryAdmin): diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py index 6c1c8726e1..e6324ad7cd 100644 --- a/ietf/utils/admin.py +++ b/ietf/utils/admin.py @@ -51,6 +51,19 @@ def _link(self): _link.admin_order_field = ordering return _link + +class SaferStackedInline(admin.StackedInline): + """StackedInline without delete by default""" + can_delete = False # no delete button + show_change_link = True # show a link to the resource (where it can be deleted) + + +class SaferTabularInline(admin.TabularInline): + """TabularInline without delete by default""" + can_delete = False # no delete button + show_change_link = True # show a link to the resource (where it can be deleted) + + from .models import DumpInfo class DumpInfoAdmin(admin.ModelAdmin): list_display = ['date', 'host', 'tz'] From 4f0102d4f48e59ab65e0545959315f060a9c3a11 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 16 Dec 2025 14:28:02 -0400 Subject: [PATCH 031/158] fix: remove redundant options (#10117) --- ietf/doc/admin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index 920e389a9a..8f26b222e1 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -33,8 +33,6 @@ class DocAuthorInline(SaferTabularInline): model = DocumentAuthor raw_id_fields = ['person', 'email'] extra = 1 - can_delete = False - show_change_link = True class DocActionHolderInline(SaferTabularInline): model = DocumentActionHolder From 4f1d8000b47aa3a3deadabfa4a863cb741227281 Mon Sep 17 00:00:00 2001 From: Orie Steele Date: Fri, 19 Dec 2025 08:38:07 -0600 Subject: [PATCH 032/158] Merge pull request #10135 from OR13/patch-1 fix: change title from 'IAB news we can use' to 'IESG Liaison News' --- ietf/iesg/agenda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/iesg/agenda.py b/ietf/iesg/agenda.py index 587713089f..ace4c9ec40 100644 --- a/ietf/iesg/agenda.py +++ b/ietf/iesg/agenda.py @@ -133,7 +133,7 @@ def agenda_sections(): ('4.2', {'title':"WG rechartering"}), ('4.2.1', {'title':"Under evaluation for IETF review", 'docs':[]}), ('4.2.2', {'title':"Proposed for approval", 'docs':[]}), - ('5', {'title':"IAB news we can use"}), + ('5', {'title':"IESG Liaison News"}), ('6', {'title':"Management issues"}), ('7', {'title':"Any Other Business (WG News, New Proposals, etc.)"}), ]) From 7f566673abef2df32c8bf477fda1f6bf63fc0cf9 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 19 Dec 2025 12:15:11 -0600 Subject: [PATCH 033/158] feat: move base containers to trixie (#9535) (#10127) * Use latest release of yanglint, e.g. libyang3-tools. * Remove apt-utils, since apt-key is removed. Fixes: #8701 Co-authored-by: avtobiff Co-authored-by: Per Andersson --- docker/base.Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 2501636049..1b1f5264b8 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-bookworm +FROM python:3.12-trixie LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive @@ -7,7 +7,7 @@ ENV NODE_MAJOR=16 # Update system packages RUN apt-get update \ && apt-get -qy upgrade \ - && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 + && apt-get -y install --no-install-recommends dialog 2>&1 # Add Node.js Source RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \ @@ -51,7 +51,6 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends libgtk2.0-0 \ libgtk-3-0 \ libnotify-dev \ - libgconf-2-4 \ libgbm-dev \ libnss3 \ libxss1 \ @@ -60,7 +59,7 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends libmagic-dev \ libmariadb-dev \ libmemcached-tools \ - libyang2-tools \ + libyang3-tools \ locales \ make \ mariadb-client \ From e510c8e4a9016c267a1b02557ebaa3656f7f13c4 Mon Sep 17 00:00:00 2001 From: rjsparks <10996692+rjsparks@users.noreply.github.com> Date: Fri, 19 Dec 2025 18:29:28 +0000 Subject: [PATCH 034/158] ci: update base image target version to 20251219T1815 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index ae59ba1440..d40a734f89 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20251201T1548 +FROM ghcr.io/ietf-tools/datatracker-app-base:20251219T1815 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 726f080c67..8b107939b6 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20251201T1548 +20251219T1815 From 6ee56b52c1b412c0ba8cacd6421a3c8b529f5a4e Mon Sep 17 00:00:00 2001 From: Absit Iniuria Date: Fri, 19 Dec 2025 19:02:42 +0000 Subject: [PATCH 035/158] fix: adjust rendering of agenda sesh pop-up on mobile (#10134) * fix: adjust popup display on mobile * fix: adjust ui spacing --- client/agenda/AgendaDetailsModal.vue | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/client/agenda/AgendaDetailsModal.vue b/client/agenda/AgendaDetailsModal.vue index 2582bf2159..69c8ef8b53 100644 --- a/client/agenda/AgendaDetailsModal.vue +++ b/client/agenda/AgendaDetailsModal.vue @@ -274,6 +274,7 @@ async function fetchSessionMaterials () { From 3565d8427efdbefb2b36cd2fe0776ad67c55d130 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 19 Dec 2025 15:51:47 -0600 Subject: [PATCH 036/158] fix: revert "feat: move base containers to trixie (#9535) (#10127)" (#10140) This reverts commit 7f566673abef2df32c8bf477fda1f6bf63fc0cf9. --- docker/base.Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 1b1f5264b8..2501636049 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-trixie +FROM python:3.12-bookworm LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive @@ -7,7 +7,7 @@ ENV NODE_MAJOR=16 # Update system packages RUN apt-get update \ && apt-get -qy upgrade \ - && apt-get -y install --no-install-recommends dialog 2>&1 + && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 # Add Node.js Source RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \ @@ -51,6 +51,7 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends libgtk2.0-0 \ libgtk-3-0 \ libnotify-dev \ + libgconf-2-4 \ libgbm-dev \ libnss3 \ libxss1 \ @@ -59,7 +60,7 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends libmagic-dev \ libmariadb-dev \ libmemcached-tools \ - libyang3-tools \ + libyang2-tools \ locales \ make \ mariadb-client \ From c85ebbf326eb5dc6fa23e315e462ae4fefffb709 Mon Sep 17 00:00:00 2001 From: rjsparks <10996692+rjsparks@users.noreply.github.com> Date: Fri, 19 Dec 2025 22:04:33 +0000 Subject: [PATCH 037/158] ci: update base image target version to 20251219T2152 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index d40a734f89..e05ad51d1b 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20251219T1815 +FROM ghcr.io/ietf-tools/datatracker-app-base:20251219T2152 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 8b107939b6..f8afdadf36 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20251219T1815 +20251219T2152 From 3b07c70435697625d5c20432f619a9f91c5c10e3 Mon Sep 17 00:00:00 2001 From: Absit Iniuria Date: Fri, 9 Jan 2026 15:59:30 +0000 Subject: [PATCH 038/158] feat: provide link to detailed submission status page for submission api (#10233) * feat: include link to detailed submission status page for submission api * chore: rename endpoint to submission_url --------- Co-authored-by: nouralmaa --- ietf/submit/tests.py | 9 ++++++++- ietf/submit/views.py | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index ede63d2752..216fc7de6b 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -2404,7 +2404,7 @@ def test_upload_draft(self): response = r.json() self.assertCountEqual( response.keys(), - ['id', 'name', 'rev', 'status_url'], + ['id', 'name', 'rev', 'status_url', 'submission_url'], ) submission_id = int(response['id']) self.assertEqual(response['name'], 'draft-somebody-test') @@ -2416,6 +2416,13 @@ def test_upload_draft(self): kwargs={'submission_id': submission_id}, ), ) + self.assertEqual( + response['submission_url'], + 'https://datatracker.example.com' + urlreverse( + 'ietf.submit.views.submission_status', + kwargs={'submission_id': submission_id}, + ) + ) self.assertEqual(mock_task.delay.call_count, 1) self.assertEqual(mock_task.delay.call_args.args, (submission_id,)) submission = Submission.objects.get(pk=submission_id) diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 8329a312bb..2db3f51098 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -182,6 +182,10 @@ def err(code, error, messages=None): settings.IDTRACKER_BASE_URL, urlreverse(api_submission_status, kwargs={'submission_id': submission.pk}), ), + 'submission_url': urljoin( + settings.IDTRACKER_BASE_URL, + urlreverse("ietf.submit.views.submission_status", kwargs={'submission_id': submission.pk}), + ), } ) else: From 967fffaa61a5bb6ba69e2e6766f043dee86799ee Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 9 Jan 2026 12:25:00 -0400 Subject: [PATCH 039/158] fix: f-strings for replicator log msgs (#10234) --- ietf/blobdb/replication.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/blobdb/replication.py b/ietf/blobdb/replication.py index b9d55c9498..d251d3b95c 100644 --- a/ietf/blobdb/replication.py +++ b/ietf/blobdb/replication.py @@ -146,11 +146,11 @@ def replicate_blob(bucket, name): blob = fetch_blob_via_sql(bucket, name) if blob is None: if verbose_logging_enabled(): - log.log("Deleting {bucket}:{name} from replica") + log.log(f"Deleting {bucket}:{name} from replica") try: destination_storage.delete(name) except Exception as e: - log.log("Failed to delete {bucket}:{name} from replica: {e}") + log.log(f"Failed to delete {bucket}:{name} from replica: {e}") raise ReplicationError from e else: # Add metadata expected by the MetadataS3Storage @@ -170,7 +170,7 @@ def replicate_blob(bucket, name): try: destination_storage.save(name, file_with_metadata) except Exception as e: - log.log("Failed to save {bucket}:{name} to replica: {e}") + log.log(f"Failed to save {bucket}:{name} to replica: {e}") raise ReplicationError from e From 79cb013a190080c3cf3cf0032253da9193d36491 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 9 Jan 2026 19:18:59 -0400 Subject: [PATCH 040/158] chore: squelch pyparsing deprecation warnings (#10239) --- ietf/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ietf/settings.py b/ietf/settings.py index f8d8a28d65..05eab0f12f 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -35,6 +35,8 @@ warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="oic.utils.time_util") warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="pytz.tzinfo") warnings.filterwarnings("ignore", message="'instantiateVariableFont' is deprecated", module="weasyprint") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="bibtexparser") # https://github.com/sciunto-org/python-bibtexparser/issues/502 +warnings.filterwarnings("ignore", category=DeprecationWarning, module="pyparsing") # https://github.com/sciunto-org/python-bibtexparser/issues/502 base_path = pathlib.Path(__file__).resolve().parent From d06001f640315502988ec6eb99ae3d6f08b3fc6c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 14 Jan 2026 13:55:12 -0400 Subject: [PATCH 041/158] feat: RPC modernization APIs (#9631) * feat: Add rpc_person API call * chore: Add OpenAPI yaml for RPC API * feat: api endpoint for docs submitted to the rpc * chore: remove some debug * feat: api for created demo people * feat: optimize fetching persons * feat: api for creating demo drafts (#6402) * fix: Typo in rpcapi.yaml * refactor: Allow existing Person in create_demo_person (#6398) * fix: include pubreq docevents in demo drafts (#6404) * fix: Minor API fixes (#6405) * chore: Document 404 from rpcapi get_person_by_id * feat: adding rev to demo doc creation (#6425) * feat: adding rev to demo doc creation * fix: remove attempt to control required * refactor: Replace api token checks with decorator (#6434) * feat: Add @requires_api_token decorator * refactor: Use @requires_api_token * refactor: Tweak api token endpoints This might be drifting from the design intent, but at least uses the defined endpoint values. Further cleanup may well be needed. * refactor: Improve usability of @requires_api_token * feat: get_draft_by_id api call (#6446) * fix: construct the cdn photo url correctly (#6491) * chore: restructure the rpc api (#6505) * fix: authenticate person api correctly * chore: Fix plain_name lookup * feat: subject_id -> Person api call (#6566) * feat: subject_id -> Person api call * doc: Add error responses to openapi spec * feat: rpc api drafts by names (#6853) * feat: get stream rfcs were first published into (#7814) * feat: get stream rfcs were first published into * chore: black * fix: deal with the case of no DocHistory having a stream * fix: return "none" from drafts_by_names if a draft has a stream of None (#7863) * feat: API changes needed for "real" draft imports (#7877) * style: black * fix: submitted as DateTime, not Date * fix: pk -> id in API * feat: rpc_draft source_format * feat: add shepherd to rpc_draft() api * fix: "unknown" src fmt instead of error * feat: add intended_std_level to api * refactor: blank, not None, for shepherd / std_level * style: black * fix: typo in drafts_by_names() (#7912) * feat: support for building rfc authors (#7983) * feat: support for building rfc authors * chore: copyrights * feat: api to gather draft authors (#8126) * feat: api to gather draft authors * chore: black * ci: tag feature branch release images * chore: fix git nonsense * feat: API for RFC metadata fetch/filtering (#8291) * feat: RFC API for rfceditor website (WIP) * feat: pagination + ordered RFC index * feat: filter by publication date * feat: stream + stream filtering * feat: DOI * feat: group/area for RFCs * feat: group/area filtering * feat: result sorting * refactor: send rfc number, not name * feat: search rfc title/abstract * style: Black * feat: add 'status' field * feat: filter by 'status' * style: remove redundant parentheses * feat: add updated_by/obsoleted_by fields * feat: add 'abstract' to rpc api * chore: fix unused/duplicate imports * chore: fix mypy lint * chore: unused import * feat: retrieve single rfc, including text (#8346) * feat: retrieve single rfc Use RFC number instead of doc PK as id * feat: include text in single-rfc response * chore: drop doc id from api response * fix: many=True for identifiers (#8425) * feat: add more rfc api fields (many stubs) * chore: adding postscript (ps) to rfc meta serializer (#8560) * fix: acknowledge not-issued in RfcStatusSlugT * feat: Add API call to get references * fix: Filter drafts * chore: Optimize the data query * test: Test for norminative references API call * chore: Fix typos in tests * chore: Fix typos * refactor: Separate demo logic (#8937) * refactor: Separate demo logic * chore: Skip tests * fix: trailing slashes for all rpc api endpoints (#8940) * chore: Add RPC references API call to OpenAPI spec (#8941) * chore: Remove line noise * chore: Add RPC references API call to OpenAPI spec * chore: Update rpcapi.yaml Co-authored-by: Robert Sparks --------- Co-authored-by: Robert Sparks * fix: Fix OpenAPI spec errors (#8943) * chore: Remove more line noise * fix: Fix OpenAPI spec errors * feat: include picture URL in rpc_person API (#9009) * feat: person search endpoint (#9062) * feat: person search endpoint * refactor: address review comments * improved naming of operation/components in API schema * reused Person schema component * added serializers_rpc.py * chore: RpcPersonSerializer -> PersonSerializer Better matches the hand-written schema. * fix: search for entire term, not word-by-word * fix: only look at name/plain in search Including ascii / ascii_short might be useful eventually, but since we only show plain_name in the response it can cause confusing results. By the same reasoning we could remove email__address as well, but that's useful and I expect we'll include email addresses in our response soon anyway. * refactor: reimplement purple API in django-rest-framework (#9097) * refactor: rpc_person -> PersonViewSet * refactor: rpc_subject_person -> SubjectPersonView * refactor: rpc_persons -> RpcPersonsView * refactor: move get_persons into PersonViewSet Changes the interface to return a list of Persons instead of a map from ID to name. * refactor: rpc_draft -> DraftViewSet * refactor: drafts_by_names -> DraftsByNameView * refactor: submitted_to_rpc -> DraftViewSet * refactor: rfc_original_stream -> RfcViewSet * refactor: rpc demo APIs -> viewset * refactor: get_draft_refs -> DraftViewSet * refactor: persons_by_email -> PersonViewSet * refactor: rfc_authors -> RfcViewSet * refactor: draft_authors -> DraftViewSet * refactor: avoid \x00 in regex validator Gets turned into a literal nul somewhere in the process of generating a schema and building a Python client for purple. This has the same effect but avoids the nul. * fix: missing arg on references() action * style: ruff, remove unused imports * style: ruff ruff * chore: remove rpcapi.yaml * refactor: move API to /api/purple Side effect is that the purple API client is named PurpleApi instead of RpcApi. * fix: get_draft_authors returns DraftWithAuthors * fix: distinguish CharField flavors * fix: no serializer validators for draft name/title This prevents at least one existing draft from being looked up. * fix: get_draft_authors works with str, not int * Revert "refactor: avoid \x00 in regex validator" This reverts commit 63f40cf2 * Revert "Revert "refactor: avoid \x00 in regex validator"" (#9111) This reverts commit d8656f470045c21542824d7b9b9be41bdcb8866d. * ci: only migrate blobdb if it is configured * feat: add email/url to purple person API (#9127) * feat: expose consensus in submission api * feat: subseries api for red (#9556) * refactor: central def of subseries doc types * feat: subseries doc API * refactor: optimize queries via prefetch Reduced 4500 to 18 queries * chore: remove debug * fix: fix serialization of draft field * refactor: clean up prefetch a bit * feat: filter by subseries type * fix: restore max_limit for RFC pagination * feat: add subseries+stub titlepage_name to rfc serializer (#9569) * feat: add subseries to RfcMetadataSerializer * feat: titlepage_name for RfcAuthorSerializer Always blank for now * chore: update copyrights * refactor: use py3.12 typing syntax * fix: renumber migrations * feat: add consensus on FullDraftSerializer * feat: add type field in serializer * feat: tag subseries API endpoints for purple (#9763) * feat: tag subseries API endpoints for purple * ruff * feat: change slugs/names (#9778) * change slugs/names * change slug names * fix: update RfcStatusSlugT * fix: remove double tag (#9787) This was meant to include the API call in both purple and red API clients. It seems this does not work, at least with some generators. Need to investigate further, but we should be able to work around it. * feat: rfc authors (#9937) * feat: rfc authors * fix: distinct rfc search results * fix: include titlepage_name in author name searches * fix: add is_editor to rfcauthor model. Adjust FK on_delete. Tweak admin. * fix: renumber migration * refactor: realistic titlepage_name in factory * refactor: comment + rename iteration var * chore: bump copyright year * chore: remove country from RfcAuthor Not planning to track this * refactor: make blank=False explicit * chore: remove country from admin * fix: author_list() for type=rfc * fix: blankable RfcAuthor.person * feat: RfcAuthor-aware document_json() * feat: limit docs to rfcs for RfcAuthor admin * test: document_json authors * fix: use author_names() for get_document_emails() * feat: suggest affiliation based on RfcAuthor * chore: revert "remove country from RfcAuthor" This reverts commit 3044d10439bed2137e28356f27250600ce7a2529. * chore: revert "remove country from admin" This reverts commit 208879368fa5be5ecf79ae67b0be636db8a7e81f. * feat: use rfcauthors for nomcom eligibility (#9629) * refactor: author-based eligibility cleanup * feat: use rfcauthor recs for nomcom eligbility * chore: remove commented code * refactor: factor out helper for testing * test: test_get_qualified_author_queryset * fix: restore a necessary import * test: fix test_elig_by_author * test: fix test_decorate_volunteers_with_qualifications * test: add comment * fix: drop test for draft-rfceditor state Attempted to limit to drafts literally in the queue, but was not a valid check when looking back in time. As a practical matter, the test is not necessary. * fix: exclude double counting, not rfc pub state * test: update test --------- Co-authored-by: Robert Sparks * fix: renumber migrations * feat: basic RfcAuthor API (#9940) * refactor: move get_rfc_authors API URL * refactor: drop format_suffixes from router Creates a bunch of API endpoints we have no intention of ever using * feat: RfcAuthor API (WIP) * fix: remove debug code * style: remove stray whitespace * fix: authors API for xfer (#9961) * fix: partial implementation of rfc authors() Does not handle RfcAuthor instances where person is None yet. * refactor: authors -> bulk_authors for URL consistency * fix: add OpenApi param definition doc_id (#9962) * fix: add OpenApi param definition doc_id * add schema def for person_id * feat: RFC publication API (#9975) * feat: API to publish RFC (WIP) Incomplete and in need of refactoring, but publishes an RFC. * feat: group / formal_languages from draft * feat: allow optional formal_languages via API Could do the same with group, but not clear it would ever be used. * feat: fill in overrides/updates * feat: subseries membership * fix: tolerate race to create related docs * fix: wrap pub in a transaction * feat: prevent re-publishing draft as RFC * chore: remove stale code * chore: remove debug * feat: RFC file upload API (WIP) Checkpointing progress before going further. * feat: specify RFC, validate file exts * feat: move uploaded files into place * feat: add replace option * fix: add rest of replace option * feat: handle ad/group more consistently * chore: remove inadvertent change * chore: drop external_url, get note from draft * refactor: clarify default value logic * refactor: ID obsoletes/updates by number * fix: handle draft-stream-editorial * feat: split unknown vs already published draft error (#10011) * feat: use RfcAuthor for red API (#10014) * refactor: combine redundant serializers * feat: edit authors via RFC update API * fix: remove all None authors, not just first (#10044) * refactor: extract update_rfcauthors() method * feat: EditedRfcAuthorsDocEvent * refactor: reduce RfcAuthor instance churn * feat: create RfcAuthor edit DocEvents * feat: handle DocumentAuthor->RfcAuthor updates * refactor: reduce code duplication * fix: remove leftover import * chore: relabel titlepage_name in DocEvent desc * feat: transaction * chore: make RfcAuthorViewset read-only This can perhaps go away entirely * style: undo accidental whitespace change * fix: actual RFC file extensions/locations + tests (#10131) * chore: update list of rfc file exts * refactor: _destination() helper * fix: .notprepped.xml -> prerelease/ subdir * refactor: better prefixed DRF Routers * test: fix references API test * test: test notify_rfc_published * test: test upload_rfc_files * chore: remove unused imports * chore: remove unused imports * chore: add a todo * fix: find pubreq event for editorial stream * chore: remove obsolete rpc demo API * chore: clean up outdated comments * feat: api key for red api (#10232) * fix: avoid over-return (#10231) * fix: avoid over-return * chore: undo accidental commit This is a separate bug fix on main; let it come in from there. * refactor: authors() -> author_persons() (#10237) * refactor: authors() -> author_persons() * refactor: select_related() a couple more places * refactor: update uses of Document.authors() * chore: remove debug * fix: typo * fix: mypy lint and minor bugs; mypy->1.11.2 (#10249) * fix: doc property authors needs refactoring (#10250) * fix: doc property needs refactoring * fix: set source for author fields * chore: comment * chore: fix random typo --------- Co-authored-by: Jennifer Richards * chore: remove completed todo, add comment * chore(dev): accept devtoken for red api --------- Co-authored-by: Robert Sparks Co-authored-by: Matthew Holloway Co-authored-by: Kesara Rathnayake Co-authored-by: Rudi Matz --- dev/build/migration-start.sh | 8 +- dev/deploy-to-container/settings_local.py | 6 +- docker/configs/settings_local.py | 5 + ietf/api/routers.py | 25 +- ietf/api/serializers_rpc.py | 609 ++++++++++++++++++ ietf/api/tests_views_rpc.py | 299 +++++++++ ietf/api/urls.py | 19 +- ietf/api/urls_rpc.py | 42 ++ ietf/api/views.py | 2 +- ietf/api/views_rpc.py | 434 +++++++++++++ ietf/community/utils.py | 13 +- ietf/doc/admin.py | 10 +- ietf/doc/api.py | 194 ++++++ ietf/doc/factories.py | 15 +- .../management/commands/reset_rfc_authors.py | 69 -- ietf/doc/management/commands/tests.py | 72 --- ...r_dochistory_title_alter_document_title.py | 41 ++ ietf/doc/migrations/0028_rfcauthor.py | 84 +++ .../0029_editedrfcauthorsdocevent.py | 30 + ...r_dochistory_title_alter_document_title.py | 41 ++ ietf/doc/models.py | 134 +++- ietf/doc/resources.py | 55 +- ietf/doc/serializers.py | 316 +++++++++ ietf/doc/tests.py | 88 ++- ietf/doc/tests_draft.py | 8 +- ietf/doc/tests_review.py | 2 +- ietf/doc/utils.py | 175 ++++- ietf/doc/views_doc.py | 19 +- ietf/doc/views_search.py | 13 +- ietf/group/models.py | 3 + ietf/group/serializers.py | 11 + ietf/ietfauth/utils.py | 2 +- ietf/ipr/views.py | 3 +- ietf/name/serializers.py | 11 + ietf/nomcom/tests.py | 194 +++++- ietf/nomcom/utils.py | 123 +++- ietf/person/models.py | 12 +- ietf/secr/telechat/tests.py | 4 +- ietf/settings.py | 2 + ietf/submit/tests.py | 2 +- ietf/submit/utils.py | 2 +- ietf/sync/rfceditor.py | 16 +- ietf/sync/tests.py | 2 +- ietf/templates/doc/document_info.html | 6 +- ietf/templates/doc/index_active_drafts.html | 2 +- ietf/templates/doc/opengraph.html | 4 +- ietf/templates/doc/review/request_info.html | 6 +- .../group/manage_review_requests.html | 6 +- ietf/utils/test_utils.py | 9 + ietf/utils/validators.py | 5 +- mypy.ini | 3 + requirements.txt | 4 +- 52 files changed, 2945 insertions(+), 315 deletions(-) create mode 100644 ietf/api/serializers_rpc.py create mode 100644 ietf/api/tests_views_rpc.py create mode 100644 ietf/api/urls_rpc.py create mode 100644 ietf/api/views_rpc.py create mode 100644 ietf/doc/api.py delete mode 100644 ietf/doc/management/commands/reset_rfc_authors.py delete mode 100644 ietf/doc/management/commands/tests.py create mode 100644 ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py create mode 100644 ietf/doc/migrations/0028_rfcauthor.py create mode 100644 ietf/doc/migrations/0029_editedrfcauthorsdocevent.py create mode 100644 ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py create mode 100644 ietf/doc/serializers.py create mode 100644 ietf/group/serializers.py create mode 100644 ietf/name/serializers.py diff --git a/dev/build/migration-start.sh b/dev/build/migration-start.sh index 901026e53b..578daf5cef 100644 --- a/dev/build/migration-start.sh +++ b/dev/build/migration-start.sh @@ -3,7 +3,11 @@ echo "Running Datatracker migrations..." ./ietf/manage.py migrate --settings=settings_local -echo "Running Blobdb migrations ..." -./ietf/manage.py migrate --settings=settings_local --database=blobdb +# Check whether the blobdb database exists - inspectdb will return a false +# status if not. +if ./ietf/manage.py inspectdb --database blobdb > /dev/null 2>&1; then + echo "Running Blobdb migrations ..." + ./ietf/manage.py migrate --settings=settings_local --database=blobdb +fi echo "Done!" diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index aacf000093..055b48d0f5 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -71,11 +71,11 @@ DE_GFM_BINARY = '/usr/local/bin/de-gfm' -# No real secrets here, these are public testing values _only_ APP_API_TOKENS = { - "ietf.api.views.ingest_email_test": ["ingestion-test-token"] + "ietf.api.red_api" : ["devtoken", "redtoken"], # Not a real secret + "ietf.api.views.ingest_email_test": ["ingestion-test-token"], # Not a real secret + "ietf.api.views_rpc" : ["devtoken"], # Not a real secret } - # OIDC configuration SITE_URL = 'https://__HOSTNAME__' diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 3ee7a4295d..e357ce3f73 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -100,3 +100,8 @@ bucket_name=f"{storagename}", ), } + +APP_API_TOKENS = { + "ietf.api.red_api" : ["devtoken", "redtoken"], # Not a real secret + "ietf.api.views_rpc" : ["devtoken"], # Not a real secret +} diff --git a/ietf/api/routers.py b/ietf/api/routers.py index 745ddaa811..99afdb242a 100644 --- a/ietf/api/routers.py +++ b/ietf/api/routers.py @@ -3,14 +3,29 @@ from django.core.exceptions import ImproperlyConfigured from rest_framework import routers -class PrefixedSimpleRouter(routers.SimpleRouter): - """SimpleRouter that adds a dot-separated prefix to its basename""" + +class PrefixedBasenameMixin: + """Mixin to add a prefix to the basename of a rest_framework BaseRouter""" def __init__(self, name_prefix="", *args, **kwargs): self.name_prefix = name_prefix if len(self.name_prefix) == 0 or self.name_prefix[-1] == ".": raise ImproperlyConfigured("Cannot use a name_prefix that is empty or ends with '.'") super().__init__(*args, **kwargs) - def get_default_basename(self, viewset): - basename = super().get_default_basename(viewset) - return f"{self.name_prefix}.{basename}" + def register(self, prefix, viewset, basename=None): + # Get the superclass "register" method from the class this is mixed-in with. + # This avoids typing issues with calling super().register() directly in a + # mixin class. + super_register = getattr(super(), "register") + if not super_register or not callable(super_register): + raise TypeError("Must mixin with superclass that has register() method") + super_register(prefix, viewset, basename=f"{self.name_prefix}.{basename}") + + +class PrefixedSimpleRouter(PrefixedBasenameMixin, routers.SimpleRouter): + """SimpleRouter that adds a dot-separated prefix to its basename""" + + +class PrefixedDefaultRouter(PrefixedBasenameMixin, routers.DefaultRouter): + """DefaultRouter that adds a dot-separated prefix to its basename""" + diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py new file mode 100644 index 0000000000..2223f04aeb --- /dev/null +++ b/ietf/api/serializers_rpc.py @@ -0,0 +1,609 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +import datetime +from pathlib import Path +from typing import Literal, Optional + +from django.db import transaction +from django.urls import reverse as urlreverse +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field +from rest_framework import serializers + +from ietf.doc.expire import move_draft_files_to_archive +from ietf.doc.models import ( + DocumentAuthor, + Document, + RelatedDocument, + State, + DocEvent, + RfcAuthor, +) +from ietf.doc.serializers import RfcAuthorSerializer +from ietf.doc.utils import ( + default_consensus, + prettify_std_name, + update_action_holders, + update_rfcauthors, +) +from ietf.group.models import Group +from ietf.name.models import StreamName, StdLevelName, FormalLanguageName +from ietf.person.models import Person +from ietf.utils import log + + +class PersonSerializer(serializers.ModelSerializer): + email = serializers.EmailField(read_only=True) + picture = serializers.URLField(source="cdn_photo_url", read_only=True) + url = serializers.SerializerMethodField( + help_text="relative URL for datatracker person page" + ) + + class Meta: + model = Person + fields = ["id", "plain_name", "email", "picture", "url"] + read_only_fields = ["id", "plain_name", "email", "picture", "url"] + + @extend_schema_field(OpenApiTypes.URI) + def get_url(self, object: Person): + return urlreverse( + "ietf.person.views.profile", + kwargs={"email_or_name": object.email_address() or object.name}, + ) + + +class EmailPersonSerializer(serializers.Serializer): + email = serializers.EmailField(source="address") + person_pk = serializers.IntegerField(source="person.pk") + name = serializers.CharField(source="person.name") + last_name = serializers.CharField(source="person.last_name") + initials = serializers.CharField(source="person.initials") + + +class LowerCaseEmailField(serializers.EmailField): + def to_representation(self, value): + return super().to_representation(value).lower() + + +class AuthorPersonSerializer(serializers.ModelSerializer): + person_pk = serializers.IntegerField(source="pk", read_only=True) + last_name = serializers.CharField() + initials = serializers.CharField() + email_addresses = serializers.ListField( + source="email_set.all", child=LowerCaseEmailField() + ) + + class Meta: + model = Person + fields = ["person_pk", "name", "last_name", "initials", "email_addresses"] + + +class RfcWithAuthorsSerializer(serializers.ModelSerializer): + authors = AuthorPersonSerializer(many=True, source="author_persons") + + class Meta: + model = Document + fields = ["rfc_number", "authors"] + + +class DraftWithAuthorsSerializer(serializers.ModelSerializer): + draft_name = serializers.CharField(source="name") + authors = AuthorPersonSerializer(many=True, source="author_persons") + + class Meta: + model = Document + fields = ["draft_name", "authors"] + + +class DocumentAuthorSerializer(serializers.ModelSerializer): + """Serializer for a Person in a response""" + + plain_name = serializers.SerializerMethodField() + + class Meta: + model = DocumentAuthor + fields = ["person", "plain_name"] + + def get_plain_name(self, document_author: DocumentAuthor) -> str: + return document_author.person.plain_name() + + +class FullDraftSerializer(serializers.ModelSerializer): + # Redefine these fields so they don't pick up the regex validator patterns. + # There seem to be some non-compliant drafts in the system! If this serializer + # is used for a writeable view, the validation will need to be added back. + name = serializers.CharField(max_length=255) + title = serializers.CharField(max_length=255) + + # Other fields we need to add / adjust + source_format = serializers.SerializerMethodField() + authors = DocumentAuthorSerializer(many=True, source="documentauthor_set") + shepherd = serializers.SerializerMethodField() + consensus = serializers.SerializerMethodField() + + class Meta: + model = Document + fields = [ + "id", + "name", + "rev", + "stream", + "title", + "pages", + "source_format", + "authors", + "shepherd", + "intended_std_level", + "consensus", + ] + + def get_consensus(self, doc: Document) -> Optional[bool]: + return default_consensus(doc) + + def get_source_format( + self, doc: Document + ) -> Literal["unknown", "xml-v2", "xml-v3", "txt"]: + submission = doc.submission() + if submission is None: + return "unknown" + if ".xml" in submission.file_types: + if submission.xml_version == "3": + return "xml-v3" + else: + return "xml-v2" + elif ".txt" in submission.file_types: + return "txt" + return "unknown" + + @extend_schema_field(OpenApiTypes.EMAIL) + def get_shepherd(self, doc: Document) -> str: + if doc.shepherd: + return doc.shepherd.formatted_ascii_email() + return "" + + +class DraftSerializer(FullDraftSerializer): + class Meta: + model = Document + fields = [ + "id", + "name", + "rev", + "stream", + "title", + "pages", + "source_format", + "authors", + ] + + +class SubmittedToQueueSerializer(FullDraftSerializer): + submitted = serializers.SerializerMethodField() + consensus = serializers.SerializerMethodField() + + class Meta: + model = Document + fields = [ + "id", + "name", + "stream", + "submitted", + "consensus", + ] + + def get_submitted(self, doc) -> Optional[datetime.datetime]: + event = doc.sent_to_rfc_editor_event() + return None if event is None else event.time + + def get_consensus(self, doc) -> Optional[bool]: + return default_consensus(doc) + + +class OriginalStreamSerializer(serializers.ModelSerializer): + stream = serializers.CharField(read_only=True, source="orig_stream_id") + + class Meta: + model = Document + fields = ["rfc_number", "stream"] + + +class ReferenceSerializer(serializers.ModelSerializer): + class Meta: + model = Document + fields = ["id", "name"] + read_only_fields = ["id", "name"] + + +class EditableRfcSerializer(serializers.ModelSerializer): + # Would be nice to reconcile this with ietf.doc.serializers.RfcSerializer. + # The purposes of that serializer (representing data for Red) and this one + # (accepting updates from Purple) are different enough that separate formats + # may be needed, but if not it'd be nice to have a single RfcSerializer that + # can serve both. + # + # For now, only handles authors + authors = RfcAuthorSerializer(many=True, min_length=1, source="rfcauthor_set") + + class Meta: + model = Document + fields = ["id", "authors"] + + def update(self, instance, validated_data): + assert isinstance(instance, Document) + authors_data = validated_data.pop("rfcauthor_set", None) + if authors_data is not None: + # Construct unsaved instances from validated author data + new_authors = [RfcAuthor(**ad) for ad in authors_data] + # Update the RFC with the new author set + with transaction.atomic(): + change_events = update_rfcauthors(instance, new_authors) + for event in change_events: + event.save() + return instance + + +class RfcPubSerializer(serializers.ModelSerializer): + """Write-only serializer for RFC publication""" + # publication-related fields + published = serializers.DateTimeField(default_timezone=datetime.timezone.utc) + draft_name = serializers.RegexField( + required=False, regex=r"^draft-[a-zA-Z0-9-]+$" + ) + draft_rev = serializers.RegexField( + required=False, regex=r"^[0-9][0-9]$" + ) + + # fields on the RFC Document that need tweaking from ModelSerializer defaults + rfc_number = serializers.IntegerField(min_value=1, required=True) + group = serializers.SlugRelatedField( + slug_field="acronym", queryset=Group.objects.all(), required=False + ) + stream = serializers.PrimaryKeyRelatedField( + queryset=StreamName.objects.filter(used=True) + ) + formal_languages = serializers.PrimaryKeyRelatedField( + many=True, + required=False, + queryset=FormalLanguageName.objects.filter(used=True), + help_text=( + "formal languages used in RFC (defaults to those from draft, send empty" + "list to override)" + ) + ) + std_level = serializers.PrimaryKeyRelatedField( + queryset=StdLevelName.objects.filter(used=True), + ) + ad = serializers.PrimaryKeyRelatedField( + queryset=Person.objects.all(), + allow_null=True, + required=False, + ) + obsoletes = serializers.SlugRelatedField( + many=True, + required=False, + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + ) + updates = serializers.SlugRelatedField( + many=True, + required=False, + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + ) + subseries = serializers.ListField( + child=serializers.RegexField( + required=False, + # pattern: no leading 0, finite length (arbitrarily set to 5 digits) + regex=r"^(bcp|std|fyi)[1-9][0-9]{0,4}$", + ) + ) + # N.b., authors is _not_ a field on Document! + authors = RfcAuthorSerializer(many=True) + + class Meta: + model = Document + fields = [ + "published", + "draft_name", + "draft_rev", + "rfc_number", + "title", + "authors", + "group", + "stream", + "abstract", + "pages", + "words", + "formal_languages", + "std_level", + "ad", + "note", + "obsoletes", + "updates", + "subseries", + ] + + def validate(self, data): + if "draft_name" in data or "draft_rev" in data: + if "draft_name" not in data: + raise serializers.ValidationError( + {"draft_name": "Missing draft_name"}, + code="invalid-draft-spec", + ) + if "draft_rev" not in data: + raise serializers.ValidationError( + {"draft_rev": "Missing draft_rev"}, + code="invalid-draft-spec", + ) + return data + + def create(self, validated_data): + """Publish an RFC""" + published = validated_data.pop("published") + draft_name = validated_data.pop("draft_name", None) + draft_rev = validated_data.pop("draft_rev", None) + obsoletes = validated_data.pop("obsoletes", []) + updates = validated_data.pop("updates", []) + subseries = validated_data.pop("subseries", []) + + system_person = Person.objects.get(name="(System)") + + # If specified, retrieve draft and extract RFC default values from it + if draft_name is None: + draft = None + defaults_from_draft = { + "group": Group.objects.get(acronym="none", type_id="individ"), + } + else: + # validation enforces that draft_name and draft_rev are both present + draft = Document.objects.filter( + type_id="draft", + name=draft_name, + rev=draft_rev, + ).first() + if draft is None: + raise serializers.ValidationError( + { + "draft_name": "No such draft", + "draft_rev": "No such draft", + }, + code="invalid-draft" + ) + elif draft.get_state_slug() == "rfc": + raise serializers.ValidationError( + { + "draft_name": "Draft already published as RFC", + }, + code="already-published-draft", + ) + defaults_from_draft = { + "ad": draft.ad, + "formal_languages": draft.formal_languages.all(), + "group": draft.group, + "note": draft.note, + } + + # Transaction to clean up if something fails + with transaction.atomic(): + # create rfc, letting validated request data override draft defaults + rfc = self._create_rfc(defaults_from_draft | validated_data) + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="published_rfc", + time=published, + by=system_person, + desc="RFC published", + ) + rfc.set_state(State.objects.get(used=True, type_id="rfc", slug="published")) + + # create updates / obsoletes relations + for obsoleted_rfc_pk in obsoletes: + RelatedDocument.objects.get_or_create( + source=rfc, target=obsoleted_rfc_pk, relationship_id="obs" + ) + for updated_rfc_pk in updates: + RelatedDocument.objects.get_or_create( + source=rfc, target=updated_rfc_pk, relationship_id="updates" + ) + + # create subseries relations + for subseries_doc_name in subseries: + ss_slug = subseries_doc_name[:3] + subseries_doc, ss_doc_created = Document.objects.get_or_create( + type_id=ss_slug, name=subseries_doc_name + ) + if ss_doc_created: + subseries_doc.docevent_set.create( + type=f"{ss_slug}_doc_created", + by=system_person, + desc=f"Created {subseries_doc_name} via publication of {rfc.name}", + ) + _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create( + relationship_id="contains", target=rfc + ) + if ss_rel_created: + subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + + + # create relation with draft and update draft state + if draft is not None: + draft_changes = [] + draft_events = [] + if draft.get_state_slug() != "rfc": + draft.set_state( + State.objects.get(used=True, type="draft", slug="rfc") + ) + move_draft_files_to_archive(draft, draft.rev) + draft_changes.append(f"changed state to {draft.get_state()}") + + r, created_relateddoc = RelatedDocument.objects.get_or_create( + source=draft, target=rfc, relationship_id="became_rfc", + ) + if created_relateddoc: + change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format( + rel_name=r.relationship.name.lower(), + pretty_draft_name=prettify_std_name(draft_name), + pretty_rfc_name=prettify_std_name(rfc.name), + ) + draft_changes.append(change) + + # Always set the "draft-iesg" state. This state should be set for all drafts, so + # log a warning if it is not set. What should happen here is that ietf stream + # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index. + # Other stream documents should normally be "idexists" and be left that way. The + # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub", + # and changes any other state to "pub". If unset, it changes it to "idexists". + # This reflects historical behavior and should probably be updated, but a migration + # of existing drafts (and validation of the change) is needed before we change the + # handling. + prev_iesg_state = draft.get_state("draft-iesg") + if prev_iesg_state is None: + log.log(f'Warning while processing {rfc.name}: {draft.name} has no "draft-iesg" state') + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists") + elif prev_iesg_state.slug not in ("pub", "idexists"): + if prev_iesg_state.slug != "rfcqueue": + log.log( + 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format( + rfc.name, draft.name, prev_iesg_state.slug + ) + ) + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub") + else: + new_iesg_state = prev_iesg_state + + if new_iesg_state != prev_iesg_state: + draft.set_state(new_iesg_state) + draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}") + e = update_action_holders(draft, prev_iesg_state, new_iesg_state) + if e: + draft_events.append(e) + + # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain. + if draft.stream != rfc.stream: + log.log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format( + rfc.name, draft.name, draft.stream, rfc.stream + )) + elif draft.stream.slug in ["iab", "irtf", "ise", "editorial"]: + stream_slug = f"draft-stream-{draft.stream.slug}" + prev_state = draft.get_state(stream_slug) + if prev_state is not None and prev_state.slug != "pub": + new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub") + draft.set_state(new_state) + draft_changes.append( + f"changed {new_state.type.label} to {new_state}" + ) + e = update_action_holders(draft, prev_state, new_state) + if e: + draft_events.append(e) + if draft_changes: + draft_events.append( + DocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=system_person, + type="sync_from_rfc_editor", + desc=f"Updated while publishing {rfc.name} ({', '.join(draft_changes)})", + ) + ) + draft.save_with_history(draft_events) + + return rfc + + def _create_rfc(self, validated_data): + authors_data = validated_data.pop("authors") + formal_languages = validated_data.pop("formal_languages", []) + # todo ad field + rfc = Document.objects.create( + type_id="rfc", + name=f"rfc{validated_data['rfc_number']}", + **validated_data, + ) + rfc.formal_languages.set(formal_languages) # list of PKs is ok + for order, author_data in enumerate(authors_data): + rfc.rfcauthor_set.create( + order=order, + **author_data, + ) + return rfc + + +class RfcFileSerializer(serializers.Serializer): + # The structure of this serializer is constrained by what openapi-generator-cli's + # python generator can correctly serialize as multipart/form-data. It does not + # handle nested serializers well (or perhaps at all). ListFields with child + # ChoiceField or RegexField do not serialize correctly. DictFields don't seem + # to work. + # + # It does seem to correctly send filenames along with FileFields, even as a child + # in a ListField, so we use that to convey the file format of each item. There + # are other options we could consider (e.g., a structured CharField) but this + # works. + allowed_extensions = ( + ".html", + ".json", + ".notprepped.xml", + ".pdf", + ".txt", + ".xml", + ) + + rfc = serializers.SlugRelatedField( + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + help_text="RFC number to which the contents belong", + ) + contents = serializers.ListField( + child=serializers.FileField( + allow_empty_file=False, + use_url=False, + ), + help_text=( + "List of content files. Filename extensions are used to identify " + "file types, but filenames are otherwise ignored." + ), + ) + replace = serializers.BooleanField( + required=False, + default=False, + help_text=( + "Replace existing files for this RFC. Defaults to false. When false, " + "if _any_ files already exist for the specified RFC the upload will be " + "rejected regardless of which files are being uploaded. When true," + "existing files will be removed and new ones will be put in place. BE" + "VERY CAREFUL WITH THIS OPTION IN PRODUCTION." + ), + ) + + def validate_contents(self, data): + found_extensions = [] + for uploaded_file in data: + if not hasattr(uploaded_file, "name"): + raise serializers.ValidationError( + "filename not specified for uploaded file", + code="missing-filename", + ) + ext = "".join(Path(uploaded_file.name).suffixes) + if ext not in self.allowed_extensions: + raise serializers.ValidationError( + f"File uploaded with invalid extension '{ext}'", + code="invalid-filename-ext", + ) + if ext in found_extensions: + raise serializers.ValidationError( + f"More than one file uploaded with extension '{ext}'", + code="duplicate-filename-ext", + ) + return data + + +class NotificationAckSerializer(serializers.Serializer): + message = serializers.CharField(default="ack") diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py new file mode 100644 index 0000000000..032b4b9495 --- /dev/null +++ b/ietf/api/tests_views_rpc.py @@ -0,0 +1,299 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from io import StringIO +from pathlib import Path +from tempfile import TemporaryDirectory + +from django.conf import settings +from django.core.files.base import ContentFile +from django.db.models import Max +from django.test.utils import override_settings +from django.urls import reverse as urlreverse + +from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory +from ietf.doc.models import RelatedDocument, Document +from ietf.group.factories import RoleFactory, GroupFactory +from ietf.person.factories import PersonFactory +from ietf.utils.test_utils import APITestCase, reload_db_objects + + +class RpcApiTests(APITestCase): + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + def test_draftviewset_references(self): + viewname = "ietf.api.purple_api.draft-references" + + # non-existent draft + bad_id = Document.objects.aggregate(unused_id=Max("id") + 100)["unused_id"] + url = urlreverse(viewname, kwargs={"doc_id": bad_id}) + # Without credentials + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + # Add credentials + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 404) + + # draft without any normative references + draft = IndividualDraftFactory() + draft = reload_db_objects(draft) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + refs = r.json() + self.assertEqual(refs, []) + + # draft without any normative references but with an informative reference + draft_foo = IndividualDraftFactory() + draft_foo = reload_db_objects(draft_foo) + RelatedDocument.objects.create( + source=draft, target=draft_foo, relationship_id="refinfo" + ) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + refs = r.json() + self.assertEqual(refs, []) + + # draft with a normative reference + draft_bar = IndividualDraftFactory() + draft_bar = reload_db_objects(draft_bar) + RelatedDocument.objects.create( + source=draft, target=draft_bar, relationship_id="refnorm" + ) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + refs = r.json() + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0]["id"], draft_bar.id) + self.assertEqual(refs[0]["name"], draft_bar.name) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + def test_notify_rfc_published(self): + url = urlreverse("ietf.api.purple_api.notify_rfc_published") + area = GroupFactory(type_id="area") + draft_ad = RoleFactory(group=area, name_id="ad").person + authors = PersonFactory.create_batch(2) + draft = WgDraftFactory(group__parent=area, authors=authors) + assert isinstance(draft, Document), "WgDraftFactory should generate a Document" + unused_rfc_number = ( + Document.objects.filter(rfc_number__isnull=False).aggregate( + unused_rfc_number=Max("rfc_number") + 1 + )["unused_rfc_number"] + or 10000 + ) + + post_data = { + "published": "2025-12-17T20:29:00Z", + "draft_name": draft.name, + "draft_rev": draft.rev, + "rfc_number": unused_rfc_number, + "title": draft.title, + "authors": [ + { + "titlepage_name": f"titlepage {author.name}", + "is_editor": False, + "person": author.pk, + "email": author.email_address(), + "affiliation": "Some Affiliation", + "country": "CA", + } + for author in authors + ], + "group": draft.group.acronym, + "stream": draft.stream_id, + "abstract": draft.abstract, + "pages": draft.pages, + "words": draft.pages * 250, + "formal_languages": [], + "std_level": "ps", + "ad": draft_ad.pk, + "note": "noted", + "obsoletes": [], + "updates": [], + "subseries": [], + } + r = self.client.post(url, data=post_data, format="json") + self.assertEqual(r.status_code, 403) + + r = self.client.post( + url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} + ) + self.assertEqual(r.status_code, 200) + rfc = Document.objects.filter(rfc_number=unused_rfc_number).first() + self.assertIsNotNone(rfc) + self.assertEqual(rfc.came_from_draft(), draft) + self.assertEqual( + rfc.docevent_set.filter( + type="published_rfc", time="2025-12-17T20:29:00Z" + ).count(), + 1, + ) + self.assertEqual(rfc.title, draft.title) + self.assertEqual(rfc.documentauthor_set.count(), 0) + self.assertEqual( + list( + rfc.rfcauthor_set.values( + "titlepage_name", + "is_editor", + "person", + "email", + "affiliation", + "country", + ) + ), + [ + { + "titlepage_name": f"titlepage {author.name}", + "is_editor": False, + "person": author.pk, + "email": author.email_address(), + "affiliation": "Some Affiliation", + "country": "CA", + } + for author in authors + ], + ) + self.assertEqual(rfc.group, draft.group) + self.assertEqual(rfc.stream, draft.stream) + self.assertEqual(rfc.abstract, draft.abstract) + self.assertEqual(rfc.pages, draft.pages) + self.assertEqual(rfc.words, draft.pages * 250) + self.assertEqual(rfc.formal_languages.count(), 0) + self.assertEqual(rfc.std_level_id, "ps") + self.assertEqual(rfc.ad, draft_ad) + self.assertEqual(rfc.note, "noted") + self.assertEqual(rfc.related_that_doc("obs"), []) + self.assertEqual(rfc.related_that_doc("updates"), []) + self.assertEqual(rfc.part_of(), []) + self.assertEqual(draft.get_state().slug, "rfc") + # todo test non-empty relationships + # todo test references (when updating that is part of the handling) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + def test_upload_rfc_files(self): + def _valid_post_data(): + """Generate a valid post data dict + + Each API call needs a fresh set of files, so don't reuse the return + value from this for multiple calls! + """ + return { + "rfc": rfc.rfc_number, + "contents": [ + ContentFile(b"This is .xml", "myfile.xml"), + ContentFile(b"This is .txt", "myfile.txt"), + ContentFile(b"This is .html", "myfile.html"), + ContentFile(b"This is .pdf", "myfile.pdf"), + ContentFile(b"This is .json", "myfile.json"), + ContentFile(b"This is .notprepped.xml", "myfile.notprepped.xml"), + ], + "replace": False, + } + + url = urlreverse("ietf.api.purple_api.upload_rfc_files") + unused_rfc_number = ( + Document.objects.filter(rfc_number__isnull=False).aggregate( + unused_rfc_number=Max("rfc_number") + 1 + )["unused_rfc_number"] + or 10000 + ) + + rfc = WgRfcFactory(rfc_number=unused_rfc_number) + assert isinstance(rfc, Document), "WgRfcFactory should generate a Document" + with TemporaryDirectory() as rfc_dir: + settings.RFC_PATH = rfc_dir # affects overridden settings + rfc_path = Path(rfc_dir) + (rfc_path / "prerelease").mkdir() + content = StringIO("XML content\n") + content.name = "myrfc.xml" + + # no api key + r = self.client.post(url, _valid_post_data(), format="multipart") + self.assertEqual(r.status_code, 403) + + # invalid RFC + r = self.client.post( + url, + _valid_post_data() | {"rfc": unused_rfc_number + 1}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + + # empty files + r = self.client.post( + url, + _valid_post_data() | { + "contents": [ + ContentFile(b"", "myfile.xml"), + ContentFile(b"", "myfile.txt"), + ContentFile(b"", "myfile.html"), + ContentFile(b"", "myfile.pdf"), + ContentFile(b"", "myfile.json"), + ContentFile(b"", "myfile.notprepped.xml"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + + # bad file type + r = self.client.post( + url, + _valid_post_data() | { + "contents": [ + ContentFile(b"Some content", "myfile.jpg"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + + # valid post + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + for suffix in [".xml", ".txt", ".html", ".pdf", ".json"]: + self.assertEqual( + (rfc_path / f"rfc{unused_rfc_number}") + .with_suffix(suffix) + .read_text(), + f"This is {suffix}", + f"{suffix} file should contain the expected content", + ) + self.assertEqual( + ( + rfc_path / "prerelease" / f"rfc{unused_rfc_number}.notprepped.xml" + ).read_text(), + "This is .notprepped.xml", + ".notprepped.xml file should contain the expected content", + ) + + # re-post with replace = False should now fail + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + + # re-post with replace = True should succeed + r = self.client.post( + url, + _valid_post_data() | {"replace": True}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) # conflict diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 04575b34cb..7a082567b8 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -1,26 +1,31 @@ # Copyright The IETF Trust 2017-2024, All Rights Reserved +from drf_spectacular.views import SpectacularAPIView + from django.conf import settings -from django.urls import include +from django.urls import include, path from django.views.generic import TemplateView from ietf import api -from ietf.doc import views_ballot +from ietf.doc import views_ballot, api as doc_api from ietf.meeting import views as meeting_views from ietf.submit import views as submit_views from ietf.utils.urls import url from . import views as api_views +from .routers import PrefixedSimpleRouter # DRF API routing - disabled until we plan to use it -# from drf_spectacular.views import SpectacularAPIView -# from django.urls import path # from ietf.person import api as person_api -# from .routers import PrefixedSimpleRouter # core_router = PrefixedSimpleRouter(name_prefix="ietf.api.core_api") # core api router # core_router.register("email", person_api.EmailViewSet) # core_router.register("person", person_api.PersonViewSet) +# todo more general name for this API? +red_router = PrefixedSimpleRouter(name_prefix="ietf.api.red_api") # red api router +red_router.register("doc", doc_api.RfcViewSet) +red_router.register("subseries", doc_api.SubseriesViewSet, basename="subseries") + api.autodiscover() urlpatterns = [ @@ -32,7 +37,9 @@ url(r'^v2/person/person', api_views.ApiV2PersonExportView.as_view()), # --- DRF API --- # path("core/", include(core_router.urls)), - # path("schema/", SpectacularAPIView.as_view()), + path("purple/", include("ietf.api.urls_rpc")), + path("red/", include(red_router.urls)), + path("schema/", SpectacularAPIView.as_view()), # # --- Custom API endpoints, sorted alphabetically --- # Email alias information for drafts diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py new file mode 100644 index 0000000000..9d41ac137f --- /dev/null +++ b/ietf/api/urls_rpc.py @@ -0,0 +1,42 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved +from django.urls import include, path + +from ietf.api import views_rpc +from ietf.api.routers import PrefixedDefaultRouter +from ietf.utils.urls import url + +router = PrefixedDefaultRouter(use_regex_path=False, name_prefix="ietf.api.purple_api") +router.include_format_suffixes = False +router.register(r"draft", views_rpc.DraftViewSet, basename="draft") +router.register(r"person", views_rpc.PersonViewSet) +router.register(r"rfc", views_rpc.RfcViewSet, basename="rfc") + +router.register( + r"rfc//authors", + views_rpc.RfcAuthorViewSet, + basename="rfc-authors", +) + +urlpatterns = [ + url(r"^doc/drafts_by_names/", views_rpc.DraftsByNamesView.as_view()), + url(r"^persons/search/", views_rpc.RpcPersonSearch.as_view()), + path( + r"rfc/publish/", + views_rpc.RfcPubNotificationView.as_view(), + name="ietf.api.purple_api.notify_rfc_published", + ), + path( + r"rfc/publish/files/", + views_rpc.RfcPubFilesView.as_view(), + name="ietf.api.purple_api.upload_rfc_files", + ), + path(r"subject//person/", views_rpc.SubjectPersonView.as_view()), +] + +# add routers at the end so individual routes can steal parts of their address +# space (e.g., ^rfc/publish/ superseding the ^rfc/ routes of RfcViewSet) +urlpatterns.extend( + [ + path("", include(router.urls)), + ] +) diff --git a/ietf/api/views.py b/ietf/api/views.py index 22523b2f17..420bc39693 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -97,7 +97,7 @@ class PersonalInformationExportView(DetailView, JsonExportMixin): def get(self, request): person = get_object_or_404(self.model, user=request.user) - expand = ['searchrule', 'documentauthor', 'ad_document_set', 'ad_dochistory_set', 'docevent', + expand = ['searchrule', 'documentauthor', 'rfcauthor', 'ad_document_set', 'ad_dochistory_set', 'docevent', 'ballotpositiondocevent', 'deletedevent', 'email_set', 'groupevent', 'role', 'rolehistory', 'iprdisclosurebase', 'iprevent', 'liaisonstatementevent', 'allowlisted', 'schedule', 'constraint', 'schedulingevent', 'message', 'sendqueue', 'nominee', 'topicfeedbacklastseen', 'alias', 'email', 'apikeys', 'personevent', diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py new file mode 100644 index 0000000000..fce174ab72 --- /dev/null +++ b/ietf/api/views_rpc.py @@ -0,0 +1,434 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved +import shutil +from pathlib import Path +from tempfile import TemporaryDirectory + +from django.conf import settings +from drf_spectacular.utils import OpenApiParameter +from rest_framework import mixins, parsers, serializers, viewsets, status +from rest_framework.decorators import action +from rest_framework.exceptions import APIException +from rest_framework.views import APIView +from rest_framework.response import Response + +from django.db.models import CharField as ModelCharField, OuterRef, Subquery, Q +from django.db.models.functions import Coalesce +from django.http import Http404 +from drf_spectacular.utils import extend_schema_view, extend_schema +from rest_framework import generics +from rest_framework.fields import CharField as DrfCharField +from rest_framework.filters import SearchFilter +from rest_framework.pagination import LimitOffsetPagination + +from ietf.api.serializers_rpc import ( + PersonSerializer, + FullDraftSerializer, + DraftSerializer, + SubmittedToQueueSerializer, + OriginalStreamSerializer, + ReferenceSerializer, + EmailPersonSerializer, + RfcWithAuthorsSerializer, + DraftWithAuthorsSerializer, + NotificationAckSerializer, RfcPubSerializer, RfcFileSerializer, + EditableRfcSerializer, +) +from ietf.doc.models import Document, DocHistory, RfcAuthor +from ietf.doc.serializers import RfcAuthorSerializer +from ietf.person.models import Email, Person + + +class Conflict(APIException): + status_code = status.HTTP_409_CONFLICT + default_detail = "Conflict." + default_code = "conflict" + + +@extend_schema_view( + retrieve=extend_schema( + operation_id="get_person_by_id", + summary="Find person by ID", + description="Returns a single person", + parameters=[ + OpenApiParameter( + name="person_id", + type=int, + location="path", + description="Person ID identifying this person.", + ), + ], + ), +) +class PersonViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): + queryset = Person.objects.all() + serializer_class = PersonSerializer + api_key_endpoint = "ietf.api.views_rpc" + lookup_url_kwarg = "person_id" + + @extend_schema( + operation_id="get_persons", + summary="Get a batch of persons", + description="Returns a list of persons matching requested ids. Omits any that are missing.", + request=list[int], + responses=PersonSerializer(many=True), + ) + @action(detail=False, methods=["post"]) + def batch(self, request): + """Get a batch of rpc person names""" + pks = request.data + return Response( + self.get_serializer(Person.objects.filter(pk__in=pks), many=True).data + ) + + @extend_schema( + operation_id="persons_by_email", + summary="Get a batch of persons by email addresses", + description=( + "Returns a list of persons matching requested ids. " + "Omits any that are missing." + ), + request=list[str], + responses=EmailPersonSerializer(many=True), + ) + @action(detail=False, methods=["post"], serializer_class=EmailPersonSerializer) + def batch_by_email(self, request): + emails = Email.objects.filter(address__in=request.data, person__isnull=False) + serializer = self.get_serializer(emails, many=True) + return Response(serializer.data) + + +class SubjectPersonView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="get_subject_person_by_id", + summary="Find person for OIDC subject by ID", + description="Returns a single person", + responses=PersonSerializer, + parameters=[ + OpenApiParameter( + name="subject_id", + type=str, + description="subject ID of person to return", + location="path", + ), + ], + ) + def get(self, request, subject_id: str): + try: + user_id = int(subject_id) + except ValueError: + raise serializers.ValidationError( + {"subject_id": "This field must be an integer value."} + ) + person = Person.objects.filter(user__pk=user_id).first() + if person: + return Response(PersonSerializer(person).data) + raise Http404 + + +class RpcLimitOffsetPagination(LimitOffsetPagination): + default_limit = 10 + max_limit = 100 + + +class SingleTermSearchFilter(SearchFilter): + """SearchFilter backend that does not split terms + + The default SearchFilter treats comma or whitespace-separated terms as individual + search terms. This backend instead searches for the exact term. + """ + + def get_search_terms(self, request): + value = request.query_params.get(self.search_param, "") + field = DrfCharField(trim_whitespace=False, allow_blank=True) + cleaned_value = field.run_validation(value) + return [cleaned_value] + + +@extend_schema_view( + get=extend_schema( + operation_id="search_person", + description="Get a list of persons, matching by partial name or email", + ), +) +class RpcPersonSearch(generics.ListAPIView): + # n.b. the OpenAPI schema for this can be generated by running + # ietf/manage.py spectacular --file spectacular.yaml + # and extracting / touching up the rpc_person_search_list operation + api_key_endpoint = "ietf.api.views_rpc" + queryset = Person.objects.all() + serializer_class = PersonSerializer + pagination_class = RpcLimitOffsetPagination + + # Searchable on all name-like fields or email addresses + filter_backends = [SingleTermSearchFilter] + search_fields = ["name", "plain", "email__address"] + + +@extend_schema_view( + retrieve=extend_schema( + operation_id="get_draft_by_id", + summary="Get a draft", + description="Returns the draft for the requested ID", + parameters=[ + OpenApiParameter( + name="doc_id", + type=int, + location="path", + description="Doc ID identifying this draft.", + ), + ], + ), + submitted_to_rpc=extend_schema( + operation_id="submitted_to_rpc", + summary="List documents ready to enter the RFC Editor Queue", + description="List documents ready to enter the RFC Editor Queue", + responses=SubmittedToQueueSerializer(many=True), + ), +) +class DraftViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): + queryset = Document.objects.filter(type_id="draft") + serializer_class = FullDraftSerializer + api_key_endpoint = "ietf.api.views_rpc" + lookup_url_kwarg = "doc_id" + + @action(detail=False, serializer_class=SubmittedToQueueSerializer) + def submitted_to_rpc(self, request): + """Return documents in datatracker that have been submitted to the RPC but are not yet in the queue + + Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. + """ + ietf_docs = Q(states__type_id="draft-iesg", states__slug__in=["ann"]) + irtf_iab_ise_docs = Q( + states__type_id__in=[ + "draft-stream-iab", + "draft-stream-irtf", + "draft-stream-ise", + ], + states__slug__in=["rfc-edit"], + ) + # TODO: Need a way to talk about editorial stream docs + docs = ( + self.get_queryset() + .filter(type_id="draft") + .filter(ietf_docs | irtf_iab_ise_docs) + ) + serializer = self.get_serializer(docs, many=True) + return Response(serializer.data) + + @extend_schema( + operation_id="get_draft_references", + summary="Get normative references to I-Ds", + description=( + "Returns the id and name of each normatively " + "referenced Internet-Draft for the given docId" + ), + parameters=[ + OpenApiParameter( + name="doc_id", + type=int, + location="path", + description="Doc ID identifying this draft.", + ), + ], + responses=ReferenceSerializer(many=True), + ) + @action(detail=True, serializer_class=ReferenceSerializer) + def references(self, request, doc_id=None): + doc = self.get_object() + serializer = self.get_serializer( + [ + reference + for reference in doc.related_that_doc("refnorm") + if reference.type_id == "draft" + ], + many=True, + ) + return Response(serializer.data) + + @extend_schema( + operation_id="get_draft_authors", + summary="Gather authors of the drafts with the given names", + description="returns a list mapping draft names to objects describing authors", + request=list[str], + responses=DraftWithAuthorsSerializer(many=True), + ) + @action(detail=False, methods=["post"], serializer_class=DraftWithAuthorsSerializer) + def bulk_authors(self, request): + drafts = self.get_queryset().filter(name__in=request.data) + serializer = self.get_serializer(drafts, many=True) + return Response(serializer.data) + + +@extend_schema_view( + rfc_original_stream=extend_schema( + operation_id="get_rfc_original_streams", + summary="Get the streams RFCs were originally published into", + description="returns a list of dicts associating an RFC with its originally published stream", + responses=OriginalStreamSerializer(many=True), + ) +) +class RfcViewSet(mixins.UpdateModelMixin, viewsets.GenericViewSet): + queryset = Document.objects.filter(type_id="rfc") + api_key_endpoint = "ietf.api.views_rpc" + lookup_field = "rfc_number" + serializer_class = EditableRfcSerializer + + @action(detail=False, serializer_class=OriginalStreamSerializer) + def rfc_original_stream(self, request): + rfcs = self.get_queryset().annotate( + orig_stream_id=Coalesce( + Subquery( + DocHistory.objects.filter(doc=OuterRef("pk")) + .exclude(stream__isnull=True) + .order_by("time") + .values_list("stream_id", flat=True)[:1] + ), + "stream_id", + output_field=ModelCharField(), + ), + ) + serializer = self.get_serializer(rfcs, many=True) + return Response(serializer.data) + + @extend_schema( + operation_id="get_rfc_authors", + summary="Gather authors of the RFCs with the given numbers", + description="returns a list mapping rfc numbers to objects describing authors", + request=list[int], + responses=RfcWithAuthorsSerializer(many=True), + ) + @action(detail=False, methods=["post"], serializer_class=RfcWithAuthorsSerializer) + def bulk_authors(self, request): + rfcs = self.get_queryset().filter(rfc_number__in=request.data) + serializer = self.get_serializer(rfcs, many=True) + return Response(serializer.data) + + +class DraftsByNamesView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="get_drafts_by_names", + summary="Get a batch of drafts by draft names", + description="returns a list of drafts with matching names", + request=list[str], + responses=DraftSerializer(many=True), + ) + def post(self, request): + names = request.data + docs = Document.objects.filter(type_id="draft", name__in=names) + return Response(DraftSerializer(docs, many=True).data) + + +class RfcAuthorViewSet(viewsets.ReadOnlyModelViewSet): + """ViewSet for RfcAuthor model + + Router needs to provide rfc_number as a kwarg + """ + api_key_endpoint = "ietf.api.views_rpc" + + queryset = RfcAuthor.objects.all() + serializer_class = RfcAuthorSerializer + lookup_url_kwarg = "author_id" + rfc_number_param = "rfc_number" + + def get_queryset(self): + return ( + super() + .get_queryset() + .filter( + document__type_id="rfc", + document__rfc_number=self.kwargs[self.rfc_number_param], + ) + ) + + +class RfcPubNotificationView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="notify_rfc_published", + summary="Notify datatracker of RFC publication", + request=RfcPubSerializer, + responses=NotificationAckSerializer, + ) + def post(self, request): + serializer = RfcPubSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + # Create RFC + serializer.save() + return Response(NotificationAckSerializer().data) + + +class RfcPubFilesView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + parser_classes = [parsers.MultiPartParser] + + def _destination(self, filename: str | Path) -> Path: + """Destination for an uploaded RFC file + + Strips any path components in filename and returns an absolute Path. + """ + rfc_path = Path(settings.RFC_PATH) + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + return rfc_path / "prerelease" / filename.name + return rfc_path / filename.name + + @extend_schema( + operation_id="upload_rfc_files", + summary="Upload files for a published RFC", + request=RfcFileSerializer, + responses=NotificationAckSerializer, + ) + def post(self, request): + serializer = RfcFileSerializer( + # many=True, + data=request.data, + ) + serializer.is_valid(raise_exception=True) + rfc = serializer.validated_data["rfc"] + uploaded_files = serializer.validated_data["contents"] # list[UploadedFile] + replace = serializer.validated_data["replace"] + dest_stem = f"rfc{rfc.rfc_number}" + + # List of files that might exist for an RFC + possible_rfc_files = [ + self._destination(dest_stem + ext) + for ext in serializer.allowed_extensions + ] + if not replace: + # this is the default: refuse to overwrite anything if not replacing + for possible_existing_file in possible_rfc_files: + if possible_existing_file.exists(): + raise Conflict( + "File(s) already exist for this RFC", + code="files-exist", + ) + + with TemporaryDirectory() as tempdir: + # Save files in a temporary directory. Use the uploaded filename + # extensions to identify files, but ignore the stems and generate our own. + files_to_move = [] # list[Path] + tmpfile_stem = Path(tempdir) / dest_stem + for upfile in uploaded_files: + uploaded_filename = Path(upfile.name) # name supplied by request + uploaded_ext = "".join(uploaded_filename.suffixes) + tempfile_path = tmpfile_stem.with_suffix(uploaded_ext) + with tempfile_path.open("wb") as dest: + for chunk in upfile.chunks(): + dest.write(chunk) + files_to_move.append(tempfile_path) + # copy files to final location, removing any existing ones first if the + # remove flag was set + if replace: + for possible_existing_file in possible_rfc_files: + possible_existing_file.unlink(missing_ok=True) + for ftm in files_to_move: + shutil.move(ftm, self._destination(ftm)) + # todo store in blob storage as well (need a bucket for RFCs) + + return Response(NotificationAckSerializer().data) diff --git a/ietf/community/utils.py b/ietf/community/utils.py index f23e8d26ab..b6137095ef 100644 --- a/ietf/community/utils.py +++ b/ietf/community/utils.py @@ -72,8 +72,10 @@ def docs_matching_community_list_rule(rule): return docs.filter(group=rule.group_id) elif rule.rule_type.startswith("state_"): return docs - elif rule.rule_type in ["author", "author_rfc"]: + elif rule.rule_type == "author": return docs.filter(documentauthor__person=rule.person) + elif rule.rule_type == "author_rfc": + return docs.filter(Q(rfcauthor__person=rule.person)|Q(rfcauthor__isnull=True,documentauthor__person=rule.person)) elif rule.rule_type == "ad": return docs.filter(ad=rule.person) elif rule.rule_type == "shepherd": @@ -122,9 +124,16 @@ def community_list_rules_matching_doc(doc): # author rules if doc.type_id == "rfc": + has_rfcauthors = doc.rfcauthor_set.exists() rules |= SearchRule.objects.filter( rule_type="author_rfc", - person__in=list(Person.objects.filter(documentauthor__document=doc)), + person__in=list( + Person.objects.filter( + Q(rfcauthor__document=doc) + if has_rfcauthors + else Q(documentauthor__document=doc) + ) + ), ) else: rules |= SearchRule.objects.filter( diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index 8f26b222e1..f082418935 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -13,7 +13,8 @@ TelechatDocEvent, BallotPositionDocEvent, ReviewRequestDocEvent, InitialReviewDocEvent, AddedMessageEvent, SubmissionDocEvent, DeletedEvent, EditedAuthorsDocEvent, DocumentURL, ReviewAssignmentDocEvent, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, - BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject ) + BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject, RfcAuthor, + EditedRfcAuthorsDocEvent) from ietf.utils.admin import SaferTabularInline from ietf.utils.validators import validate_external_resource_value @@ -174,6 +175,7 @@ def short_desc(self, obj): admin.site.register(TelechatDocEvent, DocEventAdmin) admin.site.register(InitialReviewDocEvent, DocEventAdmin) admin.site.register(EditedAuthorsDocEvent, DocEventAdmin) +admin.site.register(EditedRfcAuthorsDocEvent, DocEventAdmin) admin.site.register(IanaExpertDocEvent, DocEventAdmin) class BallotPositionDocEventAdmin(DocEventAdmin): @@ -237,3 +239,9 @@ def is_deleted(self, instance): admin.site.register(StoredObject, StoredObjectAdmin) + +class RfcAuthorAdmin(admin.ModelAdmin): + list_display = ['id', 'document', 'titlepage_name', 'person', 'email', 'affiliation', 'country', 'order'] + search_fields = ['document__name', 'titlepage_name', 'person__name', 'email__address', 'affiliation', 'country'] + raw_id_fields = ["document", "person", "email"] +admin.site.register(RfcAuthor, RfcAuthorAdmin) diff --git a/ietf/doc/api.py b/ietf/doc/api.py new file mode 100644 index 0000000000..47e7e6fffd --- /dev/null +++ b/ietf/doc/api.py @@ -0,0 +1,194 @@ +# Copyright The IETF Trust 2024-2026, All Rights Reserved +"""Doc API implementations""" + +from django.db.models import OuterRef, Subquery, Prefetch, Value, JSONField, QuerySet +from django.db.models.functions import TruncDate +from django_filters import rest_framework as filters +from rest_framework import filters as drf_filters +from rest_framework.mixins import ListModelMixin, RetrieveModelMixin +from rest_framework.pagination import LimitOffsetPagination +from rest_framework.viewsets import GenericViewSet + +from ietf.group.models import Group +from ietf.name.models import StreamName, DocTypeName +from ietf.utils.timezone import RPC_TZINFO +from .models import ( + Document, + DocEvent, + RelatedDocument, + DocumentAuthor, + SUBSERIES_DOC_TYPE_IDS, +) +from .serializers import ( + RfcMetadataSerializer, + RfcStatus, + RfcSerializer, + SubseriesDocSerializer, +) + + +class RfcLimitOffsetPagination(LimitOffsetPagination): + default_limit = 10 + max_limit = 500 + + +class RfcFilter(filters.FilterSet): + published = filters.DateFromToRangeFilter() + stream = filters.ModelMultipleChoiceFilter( + queryset=StreamName.objects.filter(used=True) + ) + group = filters.ModelMultipleChoiceFilter( + queryset=Group.objects.wgs(), + field_name="group__acronym", + to_field_name="acronym", + ) + area = filters.ModelMultipleChoiceFilter( + queryset=Group.objects.areas(), + field_name="group__parent__acronym", + to_field_name="acronym", + ) + status = filters.MultipleChoiceFilter( + choices=[(slug, slug) for slug in RfcStatus.status_slugs], + method=RfcStatus.filter, + ) + sort = filters.OrderingFilter( + fields=( + ("rfc_number", "number"), # ?sort=number / ?sort=-number + ("published", "published"), # ?sort=published / ?sort=-published + ), + ) + + +class PrefetchRelatedDocument(Prefetch): + """Prefetch via a RelatedDocument + + Prefetches following RelatedDocument relationships to other docs. By default, includes + those for which the current RFC is the `source`. If `reverse` is True, includes those + for which it is the `target` instead. Defaults to only "rfc" documents. + """ + + @staticmethod + def _get_queryset(relationship_id, reverse, doc_type_ids): + """Get queryset to use for the prefetch""" + if isinstance(doc_type_ids, str): + doc_type_ids = (doc_type_ids,) + + return RelatedDocument.objects.filter( + **{ + "relationship_id": relationship_id, + f"{'source' if reverse else 'target'}__type_id__in": doc_type_ids, + } + ).select_related("source" if reverse else "target") + + def __init__(self, to_attr, relationship_id, reverse=False, doc_type_ids="rfc"): + super().__init__( + lookup="targets_related" if reverse else "relateddocument_set", + queryset=self._get_queryset(relationship_id, reverse, doc_type_ids), + to_attr=to_attr, + ) + + +def augment_rfc_queryset(queryset: QuerySet[Document]): + return ( + queryset.select_related("std_level", "stream") + .prefetch_related( + Prefetch( + "group", + Group.objects.select_related("parent"), + ), + Prefetch( + "documentauthor_set", + DocumentAuthor.objects.select_related("email", "person"), + ), + PrefetchRelatedDocument( + to_attr="drafts", + relationship_id="became_rfc", + doc_type_ids="draft", + reverse=True, + ), + PrefetchRelatedDocument(to_attr="obsoletes", relationship_id="obs"), + PrefetchRelatedDocument( + to_attr="obsoleted_by", relationship_id="obs", reverse=True + ), + PrefetchRelatedDocument(to_attr="updates", relationship_id="updates"), + PrefetchRelatedDocument( + to_attr="updated_by", relationship_id="updates", reverse=True + ), + PrefetchRelatedDocument( + to_attr="subseries", + relationship_id="contains", + reverse=True, + doc_type_ids=SUBSERIES_DOC_TYPE_IDS, + ), + ) + .annotate( + published_datetime=Subquery( + DocEvent.objects.filter( + doc_id=OuterRef("pk"), + type="published_rfc", + ) + .order_by("-time") + .values("time")[:1] + ), + ) + .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) + .annotate( + # TODO implement these fake fields for real + see_also=Value([], output_field=JSONField()), + formats=Value(["txt", "xml"], output_field=JSONField()), + keywords=Value(["keyword"], output_field=JSONField()), + errata=Value([], output_field=JSONField()), + ) + ) + + +class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + api_key_endpoint = "ietf.api.red_api" # matches prefix in ietf/api/urls.py + lookup_field = "rfc_number" + queryset = augment_rfc_queryset( + Document.objects.filter(type_id="rfc", rfc_number__isnull=False) + ).order_by("-rfc_number") + + pagination_class = RfcLimitOffsetPagination + filter_backends = [filters.DjangoFilterBackend, drf_filters.SearchFilter] + filterset_class = RfcFilter + search_fields = ["title", "abstract"] + + def get_serializer_class(self): + if self.action == "retrieve": + return RfcSerializer + return RfcMetadataSerializer + + +class PrefetchSubseriesContents(Prefetch): + def __init__(self, to_attr): + super().__init__( + lookup="relateddocument_set", + queryset=RelatedDocument.objects.filter( + relationship_id="contains", + target__type_id="rfc", + ).prefetch_related( + Prefetch( + "target", + queryset=augment_rfc_queryset(Document.objects.all()), + ) + ), + to_attr=to_attr, + ) + + +class SubseriesFilter(filters.FilterSet): + type = filters.ModelMultipleChoiceFilter( + queryset=DocTypeName.objects.filter(pk__in=SUBSERIES_DOC_TYPE_IDS) + ) + + +class SubseriesViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + api_key_endpoint = "ietf.api.red_api" # matches prefix in ietf/api/urls.py + lookup_field = "name" + serializer_class = SubseriesDocSerializer + queryset = Document.objects.subseries_docs().prefetch_related( + PrefetchSubseriesContents(to_attr="contents") + ) + filter_backends = [filters.DjangoFilterBackend] + filterset_class = SubseriesFilter diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index 19aa9ecc9c..aad01be04f 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -14,7 +14,7 @@ from ietf.doc.models import ( Document, DocEvent, NewRevisionDocEvent, State, DocumentAuthor, StateDocEvent, BallotPositionDocEvent, BallotDocEvent, BallotType, IRSGBallotDocEvent, TelechatDocEvent, - DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, DocExtResource ) + DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, DocExtResource, RfcAuthor ) from ietf.group.models import Group from ietf.person.factories import PersonFactory from ietf.group.factories import RoleFactory @@ -382,6 +382,19 @@ class Meta: country = factory.Faker('country') order = factory.LazyAttribute(lambda o: o.document.documentauthor_set.count() + 1) +class RfcAuthorFactory(factory.django.DjangoModelFactory): + class Meta: + model = RfcAuthor + + document = factory.SubFactory(DocumentFactory) + titlepage_name = factory.LazyAttribute( + lambda obj: " ".join([obj.person.initials(), obj.person.last_name()]) + ) + person = factory.SubFactory('ietf.person.factories.PersonFactory') + email = factory.LazyAttribute(lambda obj: obj.person.email()) + affiliation = factory.Faker('company') + order = factory.LazyAttribute(lambda o: o.document.rfcauthor_set.count() + 1) + class WgDocumentAuthorFactory(DocumentAuthorFactory): document = factory.SubFactory(WgDraftFactory) diff --git a/ietf/doc/management/commands/reset_rfc_authors.py b/ietf/doc/management/commands/reset_rfc_authors.py deleted file mode 100644 index e2ab5f1208..0000000000 --- a/ietf/doc/management/commands/reset_rfc_authors.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright The IETF Trust 2024, All Rights Reserved - -# Reset an RFC's authors to those of the draft it came from -from django.core.management.base import BaseCommand, CommandError - -from ietf.doc.models import Document, DocEvent -from ietf.person.models import Person - - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument("rfcnum", type=int, help="RFC number to modify") - parser.add_argument( - "--force", - action="store_true", - help="reset even if RFC already has authors", - ) - - def handle(self, *args, **options): - try: - rfc = Document.objects.get(type="rfc", rfc_number=options["rfcnum"]) - except Document.DoesNotExist: - raise CommandError( - f"rfc{options['rfcnum']} does not exist in the Datatracker." - ) - - draft = rfc.came_from_draft() - if draft is None: - raise CommandError(f"{rfc.name} did not come from a draft. Can't reset.") - - orig_authors = rfc.documentauthor_set.all() - if orig_authors.exists(): - # Potentially dangerous, so refuse unless "--force" is specified - if not options["force"]: - raise CommandError( - f"{rfc.name} already has authors. Not resetting. Use '--force' to reset anyway." - ) - removed_auth_names = list(orig_authors.values_list("person__name", flat=True)) - rfc.documentauthor_set.all().delete() - DocEvent.objects.create( - doc=rfc, - by=Person.objects.get(name="(System)"), - type="edited_authors", - desc=f"Removed all authors: {', '.join(removed_auth_names)}", - ) - self.stdout.write( - self.style.SUCCESS( - f"Removed author(s): {', '.join(removed_auth_names)}" - ) - ) - - for author in draft.documentauthor_set.all(): - # Copy the author but point at the new doc. - # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances - author.pk = None - author.id = None - author._state.adding = True - author.document = rfc - author.save() - self.stdout.write( - self.style.SUCCESS(f"Added author {author.person.name} <{author.email}>") - ) - auth_names = draft.documentauthor_set.values_list("person__name", flat=True) - DocEvent.objects.create( - doc=rfc, - by=Person.objects.get(name="(System)"), - type="edited_authors", - desc=f"Set authors from rev {draft.rev} of {draft.name}: {', '.join(auth_names)}", - ) diff --git a/ietf/doc/management/commands/tests.py b/ietf/doc/management/commands/tests.py deleted file mode 100644 index 8244d87266..0000000000 --- a/ietf/doc/management/commands/tests.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright The IETF Trust 2024, All Rights Reserved -# -*- coding: utf-8 -*- - -from io import StringIO - -from django.core.management import call_command, CommandError - -from ietf.doc.factories import DocumentAuthorFactory, WgDraftFactory, WgRfcFactory -from ietf.doc.models import Document, DocumentAuthor -from ietf.utils.test_utils import TestCase - - -class CommandTests(TestCase): - @staticmethod - def _call_command(command_name, *args, **options): - """Call command, capturing (and suppressing) output""" - out = StringIO() - err = StringIO() - options["stdout"] = out - options["stderr"] = err - call_command(command_name, *args, **options) - return out.getvalue(), err.getvalue() - - def test_reset_rfc_authors(self): - command_name = "reset_rfc_authors" - - draft = WgDraftFactory() - DocumentAuthorFactory.create_batch(3, document=draft) - rfc = WgRfcFactory() # rfc does not yet have a draft - DocumentAuthorFactory.create_batch(3, document=rfc) - bad_rfc_num = ( - 1 - + Document.objects.filter(rfc_number__isnull=False) - .order_by("-rfc_number") - .first() - .rfc_number - ) - docauthor_fields = [ - field.name - for field in DocumentAuthor._meta.get_fields() - if field.name not in ["document", "id"] - ] - - with self.assertRaises(CommandError, msg="Cannot reset a bad RFC number"): - self._call_command(command_name, bad_rfc_num) - - with self.assertRaises(CommandError, msg="Cannot reset an RFC with no draft"): - self._call_command(command_name, rfc.rfc_number) - - with self.assertRaises(CommandError, msg="Cannot force-reset an RFC with no draft"): - self._call_command(command_name, rfc.rfc_number, "--force") - - # Link the draft to the rfc - rfc.targets_related.create(relationship_id="became_rfc", source=draft) - - with self.assertRaises(CommandError, msg="Cannot reset an RFC with authors"): - self._call_command(command_name, rfc.rfc_number) - - # Calling with force should work - self._call_command(command_name, rfc.rfc_number, "--force") - self.assertCountEqual( - draft.documentauthor_set.values(*docauthor_fields), - rfc.documentauthor_set.values(*docauthor_fields), - ) - - # Calling on an RFC with no authors should also work - rfc.documentauthor_set.all().delete() - self._call_command(command_name, rfc.rfc_number) - self.assertCountEqual( - draft.documentauthor_set.values(*docauthor_fields), - rfc.documentauthor_set.values(*docauthor_fields), - ) diff --git a/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py b/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py new file mode 100644 index 0000000000..e0d8560e6f --- /dev/null +++ b/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py @@ -0,0 +1,41 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0026_change_wg_state_descriptions"), + ] + + operations = [ + migrations.AlterField( + model_name="dochistory", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator, # type:ignore + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + migrations.AlterField( + model_name="document", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator, # type:ignore + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + ] diff --git a/ietf/doc/migrations/0028_rfcauthor.py b/ietf/doc/migrations/0028_rfcauthor.py new file mode 100644 index 0000000000..776dc22eb1 --- /dev/null +++ b/ietf/doc/migrations/0028_rfcauthor.py @@ -0,0 +1,84 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models +import django.db.models.deletion +import ietf.utils.models + + +class Migration(migrations.Migration): + dependencies = [ + ("person", "0005_alter_historicalperson_pronouns_selectable_and_more"), + ("doc", "0027_alter_dochistory_title_alter_document_title"), + ] + + operations = [ + migrations.CreateModel( + name="RfcAuthor", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("titlepage_name", models.CharField(max_length=128)), + ("is_editor", models.BooleanField(default=False)), + ( + "affiliation", + models.CharField( + blank=True, + help_text="Organization/company used by author for submission", + max_length=100, + ), + ), + ( + "country", + models.CharField( + blank=True, + help_text="Country used by author for submission", + max_length=255, + ), + ), + ("order", models.IntegerField(default=1)), + ( + "document", + ietf.utils.models.ForeignKey( + limit_choices_to={"type_id": "rfc"}, + on_delete=django.db.models.deletion.CASCADE, + to="doc.document", + ), + ), + ( + "email", + ietf.utils.models.ForeignKey( + blank=True, + help_text="Email address used by author for submission", + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="person.email", + ), + ), + ( + "person", + ietf.utils.models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="person.person", + ), + ), + ], + options={ + "ordering": ["document", "order"], + "indexes": [ + models.Index( + fields=["document", "order"], + name="doc_rfcauth_documen_6b5dc4_idx", + ) + ], + }, + ), + ] diff --git a/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py b/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py new file mode 100644 index 0000000000..60837c5cb2 --- /dev/null +++ b/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py @@ -0,0 +1,30 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0028_rfcauthor"), + ] + + operations = [ + migrations.CreateModel( + name="EditedRfcAuthorsDocEvent", + fields=[ + ( + "docevent_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="doc.docevent", + ), + ), + ], + bases=("doc.docevent",), + ), + ] diff --git a/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py b/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py new file mode 100644 index 0000000000..9ee858b2e8 --- /dev/null +++ b/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py @@ -0,0 +1,41 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0029_editedrfcauthorsdocevent"), + ] + + operations = [ + migrations.AlterField( + model_name="dochistory", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator(), + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + migrations.AlterField( + model_name="document", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator(), + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 8bb79b64ed..cce9203d09 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1,7 +1,8 @@ -# Copyright The IETF Trust 2010-2025, All Rights Reserved +# Copyright The IETF Trust 2010-2026, All Rights Reserved # -*- coding: utf-8 -*- +from collections import namedtuple import datetime import logging import os @@ -20,7 +21,11 @@ from django.core import checks from django.core.files.base import File from django.core.cache import caches -from django.core.validators import URLValidator, RegexValidator +from django.core.validators import ( + URLValidator, + RegexValidator, + ProhibitNullCharactersValidator, +) from django.urls import reverse as urlreverse from django.contrib.contenttypes.models import ContentType from django.conf import settings @@ -107,7 +112,13 @@ class DocumentInfo(models.Model): time = models.DateTimeField(default=timezone.now) # should probably have auto_now=True type = ForeignKey(DocTypeName, blank=True, null=True) # Draft, Agenda, Minutes, Charter, Discuss, Guideline, Email, Review, Issue, Wiki, External ... - title = models.CharField(max_length=255, validators=[validate_no_control_chars, ]) + title = models.CharField( + max_length=255, + validators=[ + ProhibitNullCharactersValidator(), + validate_no_control_chars, + ], + ) states = models.ManyToManyField(State, blank=True) # plain state (Active/Expired/...), IESG state, stream state tags = models.ManyToManyField(DocTagName, blank=True) # Revised ID Needed, ExternalParty, AD Followup, ... @@ -407,9 +418,55 @@ def friendly_state(self): else: return state.name + def author_names(self): + """Author names as a list of strings""" + names = [] + if self.type_id == "rfc" and self.rfcauthor_set.exists(): + for author in self.rfcauthor_set.select_related("person"): + if author.person: + names.append(author.person.name) + else: + # titlepage_name cannot be blank + names.append(author.titlepage_name) + else: + names = [ + author.person.name + for author in self.documentauthor_set.select_related("person") + ] + return names + + def author_persons_or_names(self): + """Authors as a list of named tuples with person and/or titlepage_name""" + Author = namedtuple("Author", "person titlepage_name") + persons_or_names = [] + if self.type_id=="rfc" and self.rfcauthor_set.exists(): + for author in self.rfcauthor_set.select_related("person"): + persons_or_names.append(Author(person=author.person, titlepage_name=author.titlepage_name)) + else: + for author in self.documentauthor_set.select_related("person"): + persons_or_names.append(Author(person=author.person, titlepage_name="")) + return persons_or_names + + def author_persons(self): + """Authors as a list of Persons + + Omits any RfcAuthors with a null person field. + """ + if self.type_id == "rfc" and self.rfcauthor_set.exists(): + authors_qs = self.rfcauthor_set.filter(person__isnull=False) + else: + authors_qs = self.documentauthor_set.all() + return [a.person for a in authors_qs.select_related("person")] + def author_list(self): + """List of author emails""" + author_qs = ( + self.rfcauthor_set + if self.type_id == "rfc" and self.rfcauthor_set.exists() + else self.documentauthor_set + ).select_related("email").order_by("order") best_addresses = [] - for author in self.documentauthor_set.all(): + for author in author_qs: if author.email: if author.email.active or not author.email.person: best_addresses.append(author.email.address) @@ -417,9 +474,6 @@ def author_list(self): best_addresses.append(author.email.person.email_address()) return ", ".join(best_addresses) - def authors(self): - return [ a.person for a in self.documentauthor_set.all() ] - # This, and several other ballot related functions here, assume that there is only one active ballot for a document at any point in time. # If that assumption is violated, they will only expose the most recently created ballot def ballot_open(self, ballot_type_slug): @@ -721,7 +775,14 @@ def referenced_by_rfcs_as_rfc_or_draft(self): if self.type_id == "rfc" and self.came_from_draft(): refs_to |= self.came_from_draft().referenced_by_rfcs() return refs_to - + + def sent_to_rfc_editor_event(self): + if self.stream_id == "ietf": + return self.docevent_set.filter(type="iesg_approved").order_by("-time").first() + elif self.stream_id in ["editorial", "iab", "irtf", "ise"]: + return self.docevent_set.filter(type="requested_publication").order_by("-time").first() + else: + return None class Meta: abstract = True @@ -845,6 +906,45 @@ def is_approved_downref(self): return False +class RfcAuthor(models.Model): + """Captures the authors of an RFC as represented on the RFC title page. + + This deviates from DocumentAuthor in that it does not get moved into the DocHistory + hierarchy as documents are saved. It will attempt to preserve email, country, and affiliation + from the DocumentAuthor objects associated with the draft leading to this RFC (which + may be wrong if the author moves or changes affiliation while the document is in the + queue). + + It does not, at this time, attempt to capture the authors from anything _but_ the title + page. The datatracker may know more about such authors based on information from the draft + leading to the RFC, and future work may take that into account. + + Once doc.rfcauthor_set.exists() for a doc of type `rfc`, doc.documentauthor_set should be + ignored. + """ + + document = ForeignKey( + "Document", + on_delete=models.CASCADE, + limit_choices_to={"type_id": "rfc"}, # only affects ModelForms (e.g., admin) + ) + titlepage_name = models.CharField(max_length=128, blank=False) + is_editor = models.BooleanField(default=False) + person = ForeignKey(Person, null=True, blank=True, on_delete=models.PROTECT) + email = ForeignKey(Email, help_text="Email address used by author for submission", blank=True, null=True, on_delete=models.PROTECT) + affiliation = models.CharField(max_length=100, blank=True, help_text="Organization/company used by author for submission") + country = models.CharField(max_length=255, blank=True, help_text="Country used by author for submission") + order = models.IntegerField(default=1) + + def __str__(self): + return u"%s %s (%s)" % (self.document.name, self.person, self.order) + + class Meta: + ordering=["document", "order"] + indexes=[ + models.Index(fields=["document", "order"]) + ] + class DocumentAuthorInfo(models.Model): person = ForeignKey(Person) # email should only be null for some historic documents @@ -894,7 +994,7 @@ class Meta: def role_for_doc(self): """Brief string description of this person's relationship to the doc""" roles = [] - if self.person in self.document.authors(): + if self.person in self.document.author_persons(): roles.append('Author') if self.person == self.document.ad: roles.append('Responsible AD') @@ -920,7 +1020,18 @@ def role_for_doc(self): 'invalid' ) + +SUBSERIES_DOC_TYPE_IDS = ("bcp", "fyi", "std") + + +class DocumentQuerySet(models.QuerySet): + def subseries_docs(self): + return self.filter(type_id__in=SUBSERIES_DOC_TYPE_IDS) + + class Document(StorableMixin, DocumentInfo): + objects = DocumentQuerySet.as_manager() + name = models.CharField(max_length=255, validators=[validate_docname,], unique=True) # immutable action_holders = models.ManyToManyField(Person, through=DocumentActionHolder, blank=True) @@ -1581,6 +1692,11 @@ class EditedAuthorsDocEvent(DocEvent): """ basis = models.CharField(help_text="What is the source or reasoning for the changes to the author list",max_length=255) + +class EditedRfcAuthorsDocEvent(DocEvent): + """Change to the RfcAuthor list for a document""" + + class BofreqEditorDocEvent(DocEvent): """ Capture the proponents of a BOF Request.""" editors = models.ManyToManyField('person.Person', blank=True) diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index 157a3ad556..556465a522 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -17,8 +17,9 @@ InitialReviewDocEvent, DocHistoryAuthor, BallotDocEvent, RelatedDocument, RelatedDocHistory, BallotPositionDocEvent, AddedMessageEvent, SubmissionDocEvent, ReviewRequestDocEvent, ReviewAssignmentDocEvent, EditedAuthorsDocEvent, DocumentURL, - IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, - BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject) + IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, + BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject, RfcAuthor, + EditedRfcAuthorsDocEvent) from ietf.name.resources import BallotPositionNameResource, DocTypeNameResource class BallotTypeResource(ModelResource): @@ -650,6 +651,31 @@ class Meta: api.doc.register(EditedAuthorsDocEventResource()) + +from ietf.person.resources import PersonResource +class EditedRfcAuthorsDocEventResource(ModelResource): + by = ToOneField(PersonResource, 'by') + doc = ToOneField(DocumentResource, 'doc') + docevent_ptr = ToOneField(DocEventResource, 'docevent_ptr') + class Meta: + queryset = EditedRfcAuthorsDocEvent.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'editedrfcauthorsdocevent' + ordering = ['id', ] + filtering = { + "id": ALL, + "time": ALL, + "type": ALL, + "rev": ALL, + "desc": ALL, + "by": ALL_WITH_RELATIONS, + "doc": ALL_WITH_RELATIONS, + "docevent_ptr": ALL_WITH_RELATIONS, + } +api.doc.register(EditedRfcAuthorsDocEventResource()) + + from ietf.name.resources import DocUrlTagNameResource class DocumentURLResource(ModelResource): doc = ToOneField(DocumentResource, 'doc') @@ -865,3 +891,28 @@ class Meta: "deleted": ALL, } api.doc.register(StoredObjectResource()) + + +from ietf.person.resources import EmailResource, PersonResource +class RfcAuthorResource(ModelResource): + document = ToOneField(DocumentResource, 'document') + person = ToOneField(PersonResource, 'person', null=True) + email = ToOneField(EmailResource, 'email', null=True) + class Meta: + queryset = RfcAuthor.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'rfcauthor' + ordering = ['id', ] + filtering = { + "id": ALL, + "titlepage_name": ALL, + "is_editor": ALL, + "affiliation": ALL, + "country": ALL, + "order": ALL, + "document": ALL_WITH_RELATIONS, + "person": ALL_WITH_RELATIONS, + "email": ALL_WITH_RELATIONS, + } +api.doc.register(RfcAuthorResource()) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py new file mode 100644 index 0000000000..05647d9ce1 --- /dev/null +++ b/ietf/doc/serializers.py @@ -0,0 +1,316 @@ +# Copyright The IETF Trust 2024-2026, All Rights Reserved +"""django-rest-framework serializers""" + +from dataclasses import dataclass +from typing import Literal, ClassVar + +from django.db.models.manager import BaseManager +from django.db.models.query import QuerySet +from drf_spectacular.utils import extend_schema_field +from rest_framework import serializers + +from ietf.group.serializers import GroupSerializer +from ietf.name.serializers import StreamNameSerializer +from .models import Document, DocumentAuthor, RfcAuthor + + +class RfcAuthorSerializer(serializers.ModelSerializer): + """Serializer for an RfcAuthor / DocumentAuthor in a response""" + datatracker_person_path = serializers.URLField( + source="person.get_absolute_url", + required=False, + help_text="URL for person link (relative to datatracker base URL)", + ) + + class Meta: + model = RfcAuthor + fields = [ + "titlepage_name", + "is_editor", + "person", + "email", # relies on email.pk being email.address + "affiliation", + "country", + "datatracker_person_path", + ] + + def to_representation(self, instance): + """instance -> primitive data types + + Translates a DocumentAuthor into an equivalent RfcAuthor we can use the same + serializer for either type. + """ + if isinstance(instance, DocumentAuthor): + # create a non-persisted RfcAuthor as a shim - do not save it! + document_author = instance + instance = RfcAuthor( + titlepage_name=document_author.person.plain_name(), + is_editor=False, + person=document_author.person, + email=document_author.email, + affiliation=document_author.affiliation, + country=document_author.country, + order=document_author.order, + ) + return super().to_representation(instance) + + def validate(self, data): + email = data.get("email") + if email is not None: + person = data.get("person") + if person is None: + raise serializers.ValidationError( + { + "email": "cannot have an email without a person", + }, + code="email-without-person", + ) + if email.person_id != person.pk: + raise serializers.ValidationError( + { + "email": "email must belong to person", + }, + code="email-person-mismatch", + ) + return data + + +@dataclass +class DocIdentifier: + type: Literal["doi", "issn"] + value: str + + +class DocIdentifierSerializer(serializers.Serializer): + type = serializers.ChoiceField(choices=["doi", "issn"]) + value = serializers.CharField() + + +type RfcStatusSlugT = Literal[ + "std", "ps", "ds", "bcp", "inf", "exp", "hist", "unkn", "not-issued", +] + + +@dataclass +class RfcStatus: + """Helper to extract the 'Status' from an RFC document for serialization""" + + slug: RfcStatusSlugT + + # Names that aren't just the slug itself. ClassVar annotation prevents dataclass from treating this as a field. + fancy_names: ClassVar[dict[RfcStatusSlugT, str]] = { + "std": "internet standard", + "ps": "proposed standard", + "ds": "draft standard", + "bcp": "best current practice", + "inf": "informational", + "exp": "experimental", + "hist": "historic", + "unkn": "unknown", + } + + # ClassVar annotation prevents dataclass from treating this as a field + stdlevelname_slug_map: ClassVar[dict[str, RfcStatusSlugT]] = { + "bcp": "bcp", + "ds": "ds", + "exp": "exp", + "hist": "hist", + "inf": "inf", + "std": "std", + "ps": "ps", + "unkn": "unkn", + } + + # ClassVar annotation prevents dataclass from treating this as a field + status_slugs: ClassVar[list[RfcStatusSlugT]] = sorted( + # TODO implement "not-issued" RFCs + set(stdlevelname_slug_map.values()) | {"not-issued"} + ) + + @property + def name(self): + return RfcStatus.fancy_names.get(self.slug, self.slug) + + @classmethod + def from_document(cls, doc: Document): + """Decide the status that applies to a document""" + return cls( + slug=(cls.stdlevelname_slug_map.get(doc.std_level.slug, "unkn")), + ) + + @classmethod + def filter(cls, queryset, name, value: list[RfcStatusSlugT]): + """Filter a queryset by status + + This is basically the inverse of the from_document() method. Given a status name, filter + the queryset to those in that status. The queryset should be a Document queryset. + """ + interesting_slugs = [ + stdlevelname_slug + for stdlevelname_slug, status_slug in cls.stdlevelname_slug_map.items() + if status_slug in value + ] + if len(interesting_slugs) == 0: + return queryset.none() + return queryset.filter(std_level__slug__in=interesting_slugs) + + +class RfcStatusSerializer(serializers.Serializer): + """Status serializer for a Document instance""" + + slug = serializers.ChoiceField(choices=RfcStatus.status_slugs) + name = serializers.CharField() + + def to_representation(self, instance: Document): + return super().to_representation(instance=RfcStatus.from_document(instance)) + + +class RelatedDraftSerializer(serializers.Serializer): + id = serializers.IntegerField(source="source.id") + name = serializers.CharField(source="source.name") + title = serializers.CharField(source="source.title") + + +class RelatedRfcSerializer(serializers.Serializer): + id = serializers.IntegerField(source="target.id") + number = serializers.IntegerField(source="target.rfc_number") + title = serializers.CharField(source="target.title") + + +class ReverseRelatedRfcSerializer(serializers.Serializer): + id = serializers.IntegerField(source="source.id") + number = serializers.IntegerField(source="source.rfc_number") + title = serializers.CharField(source="source.title") + + +class ContainingSubseriesSerializer(serializers.Serializer): + name = serializers.CharField(source="source.name") + type = serializers.CharField(source="source.type_id") + + +class RfcMetadataSerializer(serializers.ModelSerializer): + """Serialize metadata of an RFC""" + + RFC_FORMATS = ("xml", "txt", "html", "htmlized", "pdf", "ps") + + number = serializers.IntegerField(source="rfc_number") + published = serializers.DateField() + status = RfcStatusSerializer(source="*") + authors = serializers.SerializerMethodField() + group = GroupSerializer() + area = GroupSerializer(source="group.area", required=False) + stream = StreamNameSerializer() + identifiers = serializers.SerializerMethodField() + draft = serializers.SerializerMethodField() + obsoletes = RelatedRfcSerializer(many=True, read_only=True) + obsoleted_by = ReverseRelatedRfcSerializer(many=True, read_only=True) + updates = RelatedRfcSerializer(many=True, read_only=True) + updated_by = ReverseRelatedRfcSerializer(many=True, read_only=True) + subseries = ContainingSubseriesSerializer(many=True, read_only=True) + see_also = serializers.ListField(child=serializers.CharField(), read_only=True) + formats = serializers.MultipleChoiceField(choices=RFC_FORMATS) + keywords = serializers.ListField(child=serializers.CharField(), read_only=True) + errata = serializers.ListField(child=serializers.CharField(), read_only=True) + + class Meta: + model = Document + fields = [ + "number", + "title", + "published", + "status", + "pages", + "authors", + "group", + "area", + "stream", + "identifiers", + "obsoletes", + "obsoleted_by", + "updates", + "updated_by", + "subseries", + "see_also", + "draft", + "abstract", + "formats", + "keywords", + "errata", + ] + + + @extend_schema_field(RfcAuthorSerializer(many=True)) + def get_authors(self, doc: Document): + # If doc has any RfcAuthors, use those, otherwise fall back to DocumentAuthors + author_queryset: QuerySet[RfcAuthor] | QuerySet[DocumentAuthor] = ( + doc.rfcauthor_set.all() + if doc.rfcauthor_set.exists() + else doc.documentauthor_set.all() + ) + # RfcAuthorSerializer can deal with DocumentAuthor instances + return RfcAuthorSerializer( + instance=author_queryset, + many=True, + ).data + + @extend_schema_field(DocIdentifierSerializer(many=True)) + def get_identifiers(self, doc: Document): + identifiers = [] + if doc.rfc_number: + identifiers.append( + DocIdentifier(type="doi", value=f"10.17487/RFC{doc.rfc_number:04d}") + ) + return DocIdentifierSerializer(instance=identifiers, many=True).data + + @extend_schema_field(RelatedDraftSerializer) + def get_draft(self, object): + try: + related_doc = object.drafts[0] + except IndexError: + return None + return RelatedDraftSerializer(related_doc).data + + +class RfcSerializer(RfcMetadataSerializer): + """Serialize an RFC, including its metadata and text content if available""" + + text = serializers.CharField(allow_null=True) + + class Meta: + model = RfcMetadataSerializer.Meta.model + fields = RfcMetadataSerializer.Meta.fields + ["text"] + + +class SubseriesContentListSerializer(serializers.ListSerializer): + """ListSerializer that gets its object from item.target""" + + def to_representation(self, data): + """ + List of object instances -> List of dicts of primitive datatypes. + """ + # Dealing with nested relationships, data can be a Manager, + # so, first get a queryset from the Manager if needed + iterable = data.all() if isinstance(data, BaseManager) else data + # Serialize item.target instead of item itself + return [self.child.to_representation(item.target) for item in iterable] + + +class SubseriesContentSerializer(RfcMetadataSerializer): + """Serialize RFC contained in a subseries doc""" + + class Meta(RfcMetadataSerializer.Meta): + list_serializer_class = SubseriesContentListSerializer + + +class SubseriesDocSerializer(serializers.ModelSerializer): + """Serialize a subseries document (e.g., a BCP or STD)""" + + contents = SubseriesContentSerializer(many=True) + + class Meta: + model = Document + fields = [ + "name", + "type", + "contents", + ] diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 16dcfb7754..f92c9648e6 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -39,11 +39,15 @@ from ietf.doc.models import ( Document, DocRelationshipName, RelatedDocument, State, DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent, BallotType, EditedAuthorsDocEvent, StateType) -from ietf.doc.factories import ( DocumentFactory, DocEventFactory, CharterFactory, - ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, - IndividualRfcFactory, StateDocEventFactory, BallotPositionDocEventFactory, - BallotDocEventFactory, DocumentAuthorFactory, NewRevisionDocEventFactory, - StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory) +from ietf.doc.factories import (DocumentFactory, DocEventFactory, CharterFactory, + ConflictReviewFactory, WgDraftFactory, + IndividualDraftFactory, WgRfcFactory, + IndividualRfcFactory, StateDocEventFactory, + BallotPositionDocEventFactory, + BallotDocEventFactory, DocumentAuthorFactory, + NewRevisionDocEventFactory, + StatusChangeFactory, DocExtResourceFactory, + RgDraftFactory, BcpFactory, RfcAuthorFactory) from ietf.doc.forms import NotifyForm from ietf.doc.fields import SearchableDocumentsField from ietf.doc.utils import ( @@ -979,7 +983,7 @@ def test_edit_authors_permissions(self): # Relevant users not authorized to edit authors unauthorized_usernames = [ 'plain', - *[author.user.username for author in draft.authors()], + *[author.user.username for author in draft.author_persons()], draft.group.get_chair().person.user.username, 'ad' ] @@ -994,7 +998,7 @@ def test_edit_authors_permissions(self): self.client.logout() # Try to add an author via POST - still only the secretary should be able to do this. - orig_authors = draft.authors() + orig_authors = draft.author_persons() post_data = self.make_edit_authors_post_data( basis='permission test', authors=draft.documentauthor_set.all(), @@ -1012,12 +1016,12 @@ def test_edit_authors_permissions(self): for username in unauthorized_usernames: login_testing_unauthorized(self, username, url, method='post', request_kwargs=dict(data=post_data)) draft = Document.objects.get(pk=draft.pk) - self.assertEqual(draft.authors(), orig_authors) # ensure draft author list was not modified + self.assertEqual(draft.author_persons(), orig_authors) # ensure draft author list was not modified login_testing_unauthorized(self, 'secretary', url, method='post', request_kwargs=dict(data=post_data)) r = self.client.post(url, post_data) self.assertEqual(r.status_code, 302) draft = Document.objects.get(pk=draft.pk) - self.assertEqual(draft.authors(), orig_authors + [new_auth_person]) + self.assertEqual(draft.author_persons(), orig_authors + [new_auth_person]) def make_edit_authors_post_data(self, basis, authors): """Helper to generate edit_authors POST data for a set of authors""" @@ -1365,8 +1369,8 @@ def test_edit_authors_edit_fields(self): basis=change_reason ) - old_address = draft.authors()[0].email() - new_email = EmailFactory(person=draft.authors()[0], address=f'changed-{old_address}') + old_address = draft.author_persons()[0].email() + new_email = EmailFactory(person=draft.author_persons()[0], address=f'changed-{old_address}') post_data['author-0-email'] = new_email.address post_data['author-1-affiliation'] = 'University of Nowhere' post_data['author-2-country'] = 'Chile' @@ -1399,17 +1403,17 @@ def test_edit_authors_edit_fields(self): country_event = change_events.filter(desc__icontains='changed country').first() self.assertIsNotNone(email_event) - self.assertIn(draft.authors()[0].name, email_event.desc) + self.assertIn(draft.author_persons()[0].name, email_event.desc) self.assertIn(before[0]['email'], email_event.desc) self.assertIn(after[0]['email'], email_event.desc) self.assertIsNotNone(affiliation_event) - self.assertIn(draft.authors()[1].name, affiliation_event.desc) + self.assertIn(draft.author_persons()[1].name, affiliation_event.desc) self.assertIn(before[1]['affiliation'], affiliation_event.desc) self.assertIn(after[1]['affiliation'], affiliation_event.desc) self.assertIsNotNone(country_event) - self.assertIn(draft.authors()[2].name, country_event.desc) + self.assertIn(draft.author_persons()[2].name, country_event.desc) self.assertIn(before[2]['country'], country_event.desc) self.assertIn(after[2]['country'], country_event.desc) @@ -1863,13 +1867,63 @@ def test_document_ballot_needed_positions(self): def test_document_json(self): doc = IndividualDraftFactory() - + author = DocumentAuthorFactory(document=doc) + r = self.client.get(urlreverse("ietf.doc.views_doc.document_json", kwargs=dict(name=doc.name))) self.assertEqual(r.status_code, 200) data = r.json() - self.assertEqual(doc.name, data['name']) - self.assertEqual(doc.pages,data['pages']) + self.assertEqual(data["name"], doc.name) + self.assertEqual(data["pages"], doc.pages) + self.assertEqual( + data["authors"], + [ + { + "name": author.person.name, + "email": author.email.address, + "affiliation": author.affiliation, + } + ] + ) + def test_document_json_rfc(self): + doc = IndividualRfcFactory() + old_style_author = DocumentAuthorFactory(document=doc) + url = urlreverse("ietf.doc.views_doc.document_json", kwargs=dict(name=doc.name)) + + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + data = r.json() + self.assertEqual(data["name"], doc.name) + self.assertEqual(data["pages"], doc.pages) + self.assertEqual( + data["authors"], + [ + { + "name": old_style_author.person.name, + "email": old_style_author.email.address, + "affiliation": old_style_author.affiliation, + } + ] + ) + + new_style_author = RfcAuthorFactory(document=doc) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + data = r.json() + self.assertEqual(data["name"], doc.name) + self.assertEqual(data["pages"], doc.pages) + self.assertEqual( + data["authors"], + [ + { + "name": new_style_author.titlepage_name, + "email": new_style_author.email.address, + "affiliation": new_style_author.affiliation, + } + ] + ) + + def test_writeup(self): doc = IndividualDraftFactory(states = [('draft','active'),('draft-iesg','iesg-eva')],) diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index cebeac1f27..21a873c5c0 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -140,7 +140,7 @@ def test_change_state(self): self.assertEqual(draft.get_state_slug("draft-iesg"), "review-e") self.assertTrue(not draft.tags.filter(slug="ad-f-up")) self.assertTrue(draft.tags.filter(slug="need-rev")) - self.assertCountEqual(draft.action_holders.all(), [ad] + draft.authors()) + self.assertCountEqual(draft.action_holders.all(), [ad] + draft.author_persons()) self.assertEqual(draft.docevent_set.count(), events_before + 3) self.assertTrue("Test comment" in draft.docevent_set.all()[0].desc) self.assertTrue("Changed action holders" in draft.docevent_set.all()[1].desc) @@ -179,7 +179,7 @@ def test_pull_from_rfc_queue(self): states=[('draft-iesg','rfcqueue')], ) DocEventFactory(type='started_iesg_process',by=ad,doc=draft,rev=draft.rev,desc="Started IESG Process") - draft.action_holders.add(*(draft.authors())) + draft.action_holders.add(*(draft.author_persons())) url = urlreverse('ietf.doc.views_draft.change_state', kwargs=dict(name=draft.name)) login_testing_unauthorized(self, "secretary", url) @@ -279,7 +279,7 @@ def test_request_last_call(self): states=[('draft-iesg','ad-eval')], ) DocEventFactory(type='started_iesg_process',by=ad,doc=draft,rev=draft.rev,desc="Started IESG Process") - draft.action_holders.add(*(draft.authors())) + draft.action_holders.add(*(draft.author_persons())) self.client.login(username="secretary", password="secretary+password") url = urlreverse('ietf.doc.views_draft.change_state', kwargs=dict(name=draft.name)) @@ -1369,7 +1369,7 @@ def _test_changing_ah(action_holders, reason): _test_changing_ah([doc.ad, doc.shepherd.person], 'this is a first test') _test_changing_ah([doc.ad], 'this is a second test') - _test_changing_ah(doc.authors(), 'authors can do it, too') + _test_changing_ah(doc.author_persons(), 'authors can do it, too') _test_changing_ah([], 'clear it back out') def test_doc_change_action_holders_as_doc_manager(self): diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py index 8c1fc99ffe..82d1b5c232 100644 --- a/ietf/doc/tests_review.py +++ b/ietf/doc/tests_review.py @@ -822,7 +822,7 @@ def test_complete_review_upload_content(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertContains(r, assignment.review_request.team.list_email) - for author in assignment.review_request.doc.authors(): + for author in assignment.review_request.doc.author_persons(): self.assertContains(r, author.formatted_email()) # faulty post diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 2bd9a3d314..0715471551 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -13,7 +13,7 @@ from dataclasses import dataclass from hashlib import sha384 from pathlib import Path -from typing import Iterator, Optional, Union +from typing import Iterator, Optional, Union, Iterable from zoneinfo import ZoneInfo from django.conf import settings @@ -33,7 +33,14 @@ from ietf.community.models import CommunityList from ietf.community.utils import docs_tracked_by_community_list -from ietf.doc.models import Document, DocHistory, State, DocumentAuthor, DocHistoryAuthor +from ietf.doc.models import ( + DocHistory, + DocHistoryAuthor, + Document, + DocumentAuthor, + RfcAuthor, + State, EditedRfcAuthorsDocEvent, +) from ietf.doc.models import RelatedDocument, RelatedDocHistory, BallotType, DocReminder from ietf.doc.models import DocEvent, ConsensusDocEvent, BallotDocEvent, IRSGBallotDocEvent, NewRevisionDocEvent, StateDocEvent from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent, BallotPositionDocEvent @@ -534,7 +541,7 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None, doc.action_holders.clear() if tags.removed("need-rev"): # Removed the 'need-rev' tag - drop authors from the action holders list - DocumentActionHolder.objects.filter(document=doc, person__in=doc.authors()).delete() + DocumentActionHolder.objects.filter(document=doc, person__in=doc.author_persons()).delete() elif tags.added("need-rev"): # Remove the AD if we're asking for a new revision DocumentActionHolder.objects.filter(document=doc, person=doc.ad).delete() @@ -549,7 +556,7 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None, doc.action_holders.add(doc.ad) # Authors get the action if a revision is needed if tags.added("need-rev"): - for auth in doc.authors(): + for auth in doc.author_persons(): doc.action_holders.add(auth) # Now create an event if we changed the set @@ -561,6 +568,40 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None, ) +def _change_field_and_describe( + author: DocumentAuthor | RfcAuthor, + field: str, + newval, + field_display_name: str | None = None, +): + # make the change + oldval = getattr(author, field) + setattr(author, field, newval) + + was_empty = oldval is None or len(str(oldval)) == 0 + now_empty = newval is None or len(str(newval)) == 0 + + # describe the change + if oldval == newval: + return None + else: + if field_display_name is None: + field_display_name = field + + if was_empty and not now_empty: + return 'set {field} to "{new}"'.format( + field=field_display_name, new=newval + ) + elif now_empty and not was_empty: + return 'cleared {field} (was "{old}")'.format( + field=field_display_name, old=oldval + ) + else: + return 'changed {field} from "{old}" to "{new}"'.format( + field=field_display_name, old=oldval, new=newval + ) + + def update_documentauthors(doc, new_docauthors, by=None, basis=None): """Update the list of authors for a document @@ -573,27 +614,6 @@ def update_documentauthors(doc, new_docauthors, by=None, basis=None): used. These objects will not be saved, their attributes will be used to create new DocumentAuthor instances. (The document and order fields will be ignored.) """ - def _change_field_and_describe(auth, field, newval): - # make the change - oldval = getattr(auth, field) - setattr(auth, field, newval) - - was_empty = oldval is None or len(str(oldval)) == 0 - now_empty = newval is None or len(str(newval)) == 0 - - # describe the change - if oldval == newval: - return None - else: - if was_empty and not now_empty: - return 'set {field} to "{new}"'.format(field=field, new=newval) - elif now_empty and not was_empty: - return 'cleared {field} (was "{old}")'.format(field=field, old=oldval) - else: - return 'changed {field} from "{old}" to "{new}"'.format( - field=field, old=oldval, new=newval - ) - persons = [] changes = [] # list of change descriptions @@ -637,6 +657,111 @@ def _change_field_and_describe(auth, field, newval): ) for change in changes ] + +def update_rfcauthors( + rfc: Document, new_rfcauthors: Iterable[RfcAuthor], by: Person | None = None +) -> Iterable[EditedRfcAuthorsDocEvent]: + def _find_matching_author( + author_to_match: RfcAuthor, existing_authors: Iterable[RfcAuthor] + ) -> RfcAuthor | None: + """Helper to find a matching existing author""" + if author_to_match.person_id is not None: + for candidate in existing_authors: + if candidate.person_id == author_to_match.person_id: + return candidate + return None # no match + # author does not have a person, match on titlepage name + for candidate in existing_authors: + if candidate.titlepage_name == author_to_match.titlepage_name: + return candidate + return None # no match + + def _rfcauthor_from_documentauthor(docauthor: DocumentAuthor) -> RfcAuthor: + """Helper to create an equivalent RfcAuthor from a DocumentAuthor""" + return RfcAuthor( + document_id=docauthor.document_id, + titlepage_name=docauthor.person.plain_name(), # closest thing we have + is_editor=False, + person_id=docauthor.person_id, + affiliation=docauthor.affiliation, + country=docauthor.country, + order=docauthor.order, + ) + + # Is this the first time this document is getting an RfcAuthor? If so, the + # updates will need to account for the model change. + converting_from_docauthors = not rfc.rfcauthor_set.exists() + + if converting_from_docauthors: + original_authors = [ + _rfcauthor_from_documentauthor(da) for da in rfc.documentauthor_set.all() + ] + else: + original_authors = list(rfc.rfcauthor_set.all()) + + authors_to_commit = [] + changes = [] + for order, new_author in enumerate(new_rfcauthors): + matching_author = _find_matching_author(new_author, original_authors) + if matching_author is not None: + # Update existing matching author using new_author data + authors_to_commit.append(matching_author) + original_authors.remove(matching_author) # avoid reuse + # Describe changes to this author + author_changes = [] + # Update fields other than order + for field in ["titlepage_name", "is_editor", "affiliation", "country"]: + author_changes.append( + _change_field_and_describe( + matching_author, + field, + getattr(new_author, field), + # List titlepage_name as "name" in logs + "name" if field == "titlepage_name" else field, + ) + ) + # Update order + author_changes.append( + _change_field_and_describe(matching_author, "order", order + 1) + ) + matching_author.save() + author_change_summary = ", ".join( + [ch for ch in author_changes if ch is not None] + ) + if len(author_change_summary) > 0: + changes.append( + 'Changed author "{name}": {summary}'.format( + name=matching_author.titlepage_name, + summary=author_change_summary, + ) + ) + else: + # No author matched, so update the new_author and use that + new_author.document = rfc + new_author.order = order + 1 + new_author.save() + changes.append(f'Added "{new_author.titlepage_name}" as author') + # Any authors left in original_authors are no longer in the list, so remove them + for removed_author in original_authors: + # Skip actual removal of old authors if we are converting from the + # DocumentAuthor models - the original_authors were just stand-ins anyway. + if not converting_from_docauthors: + removed_author.delete() + changes.append(f'Removed "{removed_author.titlepage_name}" as author') + # Create DocEvents, but leave it up to caller to save + if by is None: + by = Person.objects.get(name="(System)") + return [ + EditedRfcAuthorsDocEvent( + type="edited_authors", + by=by, + doc=rfc, + desc=change, + ) + for change in changes + ] + + def update_reminder(doc, reminder_type_slug, event, due_date): reminder_type = DocReminderTypeName.objects.get(slug=reminder_type_slug) diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 5564904504..0578da1b77 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -1653,11 +1653,18 @@ def extract_name(s): data["state"] = extract_name(doc.get_state()) data["intended_std_level"] = extract_name(doc.intended_std_level) data["std_level"] = extract_name(doc.std_level) + author_qs = ( + doc.rfcauthor_set + if doc.type_id == "rfc" and doc.rfcauthor_set.exists() + else doc.documentauthor_set + ).select_related("person", "email").order_by("order") data["authors"] = [ - dict(name=author.person.name, - email=author.email.address if author.email else None, - affiliation=author.affiliation) - for author in doc.documentauthor_set.all().select_related("person", "email").order_by("order") + { + "name": author.titlepage_name if hasattr(author, "titlepage_name") else author.person.name, + "email": author.email.address if author.email else None, + "affiliation": author.affiliation, + } + for author in author_qs ] data["shepherd"] = doc.shepherd.formatted_email() if doc.shepherd else None data["ad"] = doc.ad.role_email("ad").formatted_email() if doc.ad else None @@ -1941,9 +1948,9 @@ def edit_action_holders(request, name): role_ids = dict() # maps role slug to list of Person IDs (assumed numeric in the JavaScript) extra_prefetch = [] # list of Person objects to prefetch for select2 field - if len(doc.authors()) > 0: + authors = doc.author_persons() + if len(authors) > 0: doc_role_labels.append(dict(slug='authors', label='Authors')) - authors = doc.authors() role_ids['authors'] = [p.pk for p in authors] extra_prefetch += authors diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 3b67061b05..4232d77f6c 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -219,7 +219,7 @@ def retrieve_search_results(form, all_types=False): queries.extend([Q(targets_related__source__name__icontains=look_for, targets_related__relationship_id="became_rfc")]) combined_query = reduce(operator.or_, queries) - docs = docs.filter(combined_query).distinct() + docs = docs.filter(combined_query) # rfc/active/old check buttons allowed_draft_states = [] @@ -229,20 +229,23 @@ def retrieve_search_results(form, all_types=False): allowed_draft_states.extend(['repl', 'expired', 'auth-rm', 'ietf-rm']) docs = docs.filter(Q(states__slug__in=allowed_draft_states) | - ~Q(type__slug='draft')).distinct() + ~Q(type__slug='draft')) # radio choices by = query["by"] if by == "author": docs = docs.filter( Q(documentauthor__person__alias__name__icontains=query["author"]) | - Q(documentauthor__person__email__address__icontains=query["author"]) + Q(documentauthor__person__email__address__icontains=query["author"]) | + Q(rfcauthor__person__alias__name__icontains=query["author"]) | + Q(rfcauthor__person__email__address__icontains=query["author"]) | + Q(rfcauthor__titlepage_name__icontains=query["author"]) ) elif by == "group": docs = docs.filter(group__acronym__iexact=query["group"]) elif by == "area": docs = docs.filter(Q(group__type="wg", group__parent=query["area"]) | - Q(group=query["area"])).distinct() + Q(group=query["area"])) elif by == "ad": docs = docs.filter(ad=query["ad"]) elif by == "state": @@ -255,6 +258,8 @@ def retrieve_search_results(form, all_types=False): elif by == "stream": docs = docs.filter(stream=query["stream"]) + docs=docs.distinct() + return docs diff --git a/ietf/group/models.py b/ietf/group/models.py index 2d5e7c4e6f..a7e3c6616e 100644 --- a/ietf/group/models.py +++ b/ietf/group/models.py @@ -111,6 +111,9 @@ def active_wgs(self): def closed_wgs(self): return self.wgs().exclude(state__in=Group.ACTIVE_STATE_IDS) + def areas(self): + return self.get_queryset().filter(type="area") + def with_meetings(self): return self.get_queryset().filter(type__features__has_meetings=True) diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py new file mode 100644 index 0000000000..08e6bba81a --- /dev/null +++ b/ietf/group/serializers.py @@ -0,0 +1,11 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +"""django-rest-framework serializers""" +from rest_framework import serializers + +from .models import Group + + +class GroupSerializer(serializers.ModelSerializer): + class Meta: + model = Group + fields = ["acronym", "name", "type"] diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py index 1f634278be..0df667fbd2 100644 --- a/ietf/ietfauth/utils.py +++ b/ietf/ietfauth/utils.py @@ -287,7 +287,7 @@ def is_individual_draft_author(user, doc): if not hasattr(user, 'person'): return False - if user.person in doc.authors(): + if user.person in doc.author_persons(): return True return False diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py index 665c99dc43..0a43ff2c27 100644 --- a/ietf/ipr/views.py +++ b/ietf/ipr/views.py @@ -81,7 +81,8 @@ def get_document_emails(ipr): addrs = gather_address_lists('ipr_posted_on_doc',doc=doc).as_strings(compact=False) - author_names = ', '.join(a.person.name for a in doc.documentauthor_set.select_related("person")) + # Get a list of author names for the salutation in the body of the email + author_names = ', '.join(doc.author_names()) context = dict( settings=settings, diff --git a/ietf/name/serializers.py b/ietf/name/serializers.py new file mode 100644 index 0000000000..a764f56051 --- /dev/null +++ b/ietf/name/serializers.py @@ -0,0 +1,11 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +"""django-rest-framework serializers""" +from rest_framework import serializers + +from .models import StreamName + + +class StreamNameSerializer(serializers.ModelSerializer): + class Meta: + model = StreamName + fields = ["slug", "name", "desc"] diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index dcdb9ef836..b6e8c57da7 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -1,5 +1,4 @@ -# Copyright The IETF Trust 2012-2023, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2012-2025, All Rights Reserved import datetime @@ -27,8 +26,14 @@ from ietf.api.views import EmailIngestionError from ietf.dbtemplate.factories import DBTemplateFactory from ietf.dbtemplate.models import DBTemplate -from ietf.doc.factories import DocEventFactory, WgDocumentAuthorFactory, \ - NewRevisionDocEventFactory, DocumentAuthorFactory +from ietf.doc.factories import ( + DocEventFactory, + WgDocumentAuthorFactory, + NewRevisionDocEventFactory, + DocumentAuthorFactory, + RfcAuthorFactory, + WgDraftFactory, WgRfcFactory, +) from ietf.group.factories import GroupFactory, GroupHistoryFactory, RoleFactory, RoleHistoryFactory from ietf.group.models import Group, Role from ietf.meeting.factories import MeetingFactory, AttendedFactory, RegistrationFactory @@ -45,10 +50,20 @@ nomcom_kwargs_for_year, provide_private_key_to_test_client, \ key from ietf.nomcom.tasks import send_nomcom_reminders_task -from ietf.nomcom.utils import get_nomcom_by_year, make_nomineeposition, \ - get_hash_nominee_position, is_eligible, list_eligible, \ - get_eligibility_date, suggest_affiliation, ingest_feedback_email, \ - decorate_volunteers_with_qualifications, send_reminders, _is_time_to_send_reminder +from ietf.nomcom.utils import ( + get_nomcom_by_year, + make_nomineeposition, + get_hash_nominee_position, + is_eligible, + list_eligible, + get_eligibility_date, + suggest_affiliation, + ingest_feedback_email, + decorate_volunteers_with_qualifications, + send_reminders, + _is_time_to_send_reminder, + get_qualified_author_queryset, +) from ietf.person.factories import PersonFactory, EmailFactory from ietf.person.models import Email, Person from ietf.utils.mail import outbox, empty_outbox, get_payload_text @@ -2440,6 +2455,86 @@ def test_get_eligibility_date(self): NomComFactory(group__acronym=f'nomcom{this_year}', first_call_for_volunteers=datetime.date(this_year,5,6)) self.assertEqual(get_eligibility_date(),datetime.date(this_year,5,6)) + def test_get_qualified_author_queryset(self): + """get_qualified_author_queryset implements the eligiblity rules correctly + + This is not an exhaustive test of corner cases. Overlaps considerably with + rfc8989EligibilityTests.test_elig_by_author(). + """ + people = PersonFactory.create_batch(2) + extra_person = PersonFactory() + base_qs = Person.objects.filter(pk__in=[person.pk for person in people]) + now = datetime.datetime.now(tz=datetime.UTC) + one_year = datetime.timedelta(days=365) + + # Authors with no qualifying drafts + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), [] + ) + + # Authors with one qualifying draft + approved_draft = WgDraftFactory(authors=people, states=[("draft", "active")]) + DocEventFactory( + type="iesg_approved", + doc=approved_draft, + time=now - 4 * one_year, + ) + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), [] + ) + + # Create a draft that was published into an RFC. Give it an extra author who + # should not be eligible. + published_draft = WgDraftFactory(authors=people, states=[("draft", "rfc")]) + DocEventFactory( + type="iesg_approved", + doc=published_draft, + time=now - 5.5 * one_year, # < 6 years ago + ) + rfc = WgRfcFactory( + authors=people + [extra_person], + group=published_draft.group, + ) + DocEventFactory( + type="published_rfc", + doc=rfc, + time=now - 0.5 * one_year, # < 1 year ago + ) + # Period 6 years ago to 1 year ago - authors are eligible due to the + # iesg-approved draft in this window + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 6 * one_year, now - one_year), + people, + ) + + # Period 5 years ago to now - authors are eligible due to the RFC publication + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), + people, + ) + + # Use the extra_person to check that a single doc can't count both as an + # RFC _and_ an approved draft. Use an eligibility interval that includes both + # the approval and the RFC publication + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 6 * one_year, now), + people, # does not include extra_person! + ) + + # Now add an RfcAuthor for only one of the two authors to the RFC. This should + # remove the other author from the eligibility list because the DocumentAuthor + # records are no longer used. + RfcAuthorFactory( + document=rfc, + person=people[0], + titlepage_name="P. Zero", + email=people[0].email_set.first(), + ) + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), + [people[0]], + ) + class rfc8713EligibilityTests(TestCase): @@ -2724,33 +2819,41 @@ def test_elig_by_author(self): ineligible = set() p = PersonFactory() - ineligible.add(p) - + ineligible.add(p) # no RFCs or iesg-approved drafts p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=middle_date) - ineligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=middle_date) + ineligible.add(p) # only one RFC p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) + da = WgDocumentAuthorFactory( + person=p, + document__states=[("draft", "active"), ("draft-rfceditor", "ref")], + ) DocEventFactory(type='iesg_approved',doc=da.document,time=last_date) - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=first_date) - eligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=first_date) + eligible.add(p) # one RFC and one iesg-approved draft p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) + da = WgDocumentAuthorFactory( + person=p, + document__states=[("draft", "active"), ("draft-rfceditor", "ref")], + ) DocEventFactory(type='iesg_approved',doc=da.document,time=middle_date) - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=day_before_first_date) - ineligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=day_before_first_date) + ineligible.add(p) # RFC is out of the eligibility window p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) + da = WgDocumentAuthorFactory( + person=p, + document__states=[("draft", "active"), ("draft-rfceditor", "ref")], + ) DocEventFactory(type='iesg_approved',doc=da.document,time=day_after_last_date) - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=middle_date) - ineligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=middle_date) + ineligible.add(p) # iesg approval is outside the eligibility window for person in eligible: self.assertTrue(is_eligible(person,nomcom)) @@ -2878,15 +2981,38 @@ def test_volunteer(self): def test_suggest_affiliation(self): person = PersonFactory() - self.assertEqual(suggest_affiliation(person), '') - da = DocumentAuthorFactory(person=person,affiliation='auth_affil') + self.assertEqual(suggest_affiliation(person), "") + rfc_da = DocumentAuthorFactory( + person=person, + document__type_id="rfc", + affiliation="", + ) + rfc = rfc_da.document + DocEventFactory(doc=rfc, type="published_rfc") + self.assertEqual(suggest_affiliation(person), "") + + rfc_da.affiliation = "rfc_da_affil" + rfc_da.save() + self.assertEqual(suggest_affiliation(person), "rfc_da_affil") + + rfc_ra = RfcAuthorFactory(person=person, document=rfc, affiliation="") + self.assertEqual(suggest_affiliation(person), "") + + rfc_ra.affiliation = "rfc_ra_affil" + rfc_ra.save() + self.assertEqual(suggest_affiliation(person), "rfc_ra_affil") + + da = DocumentAuthorFactory(person=person, affiliation="auth_affil") NewRevisionDocEventFactory(doc=da.document) - self.assertEqual(suggest_affiliation(person), 'auth_affil') + self.assertEqual(suggest_affiliation(person), "auth_affil") + nc = NomComFactory() - nc.volunteer_set.create(person=person,affiliation='volunteer_affil') - self.assertEqual(suggest_affiliation(person), 'volunteer_affil') - RegistrationFactory(person=person, affiliation='meeting_affil') - self.assertEqual(suggest_affiliation(person), 'meeting_affil') + nc.volunteer_set.create(person=person, affiliation="volunteer_affil") + self.assertEqual(suggest_affiliation(person), "volunteer_affil") + + RegistrationFactory(person=person, affiliation="meeting_affil") + self.assertEqual(suggest_affiliation(person), "meeting_affil") + class VolunteerDecoratorUnitTests(TestCase): def test_decorate_volunteers_with_qualifications(self): @@ -2922,10 +3048,10 @@ def test_decorate_volunteers_with_qualifications(self): author_person = PersonFactory() for i in range(2): - da = WgDocumentAuthorFactory(person=author_person) + doc = WgRfcFactory(authors=[author_person]) DocEventFactory( type='published_rfc', - doc=da.document, + doc=doc, time=datetime.datetime( elig_date.year - 3, elig_date.month, diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index dd651c2941..a2ab680df6 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -18,7 +18,7 @@ from email.utils import parseaddr from textwrap import dedent -from django.db.models import Q, Count +from django.db.models import Q, Count, F, QuerySet from django.conf import settings from django.contrib.sites.models import Site from django.core.exceptions import ObjectDoesNotExist @@ -27,7 +27,7 @@ from django.shortcuts import get_object_or_404 from ietf.dbtemplate.models import DBTemplate -from ietf.doc.models import DocEvent, NewRevisionDocEvent +from ietf.doc.models import DocEvent, NewRevisionDocEvent, Document from ietf.group.models import Group, Role from ietf.person.models import Email, Person from ietf.mailtrigger.utils import gather_address_lists @@ -576,6 +576,70 @@ def get_8989_eligibility_querysets(date, base_qs): def get_9389_eligibility_querysets(date, base_qs): return get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable=three_of_five_eligible_9389) + +def get_qualified_author_queryset( + base_qs: QuerySet[Person], + eligibility_period_start: datetime.datetime, + eligibility_period_end: datetime.datetime, +): + """Filter a Person queryset, keeping those qualified by RFC 8989's author path + + The author path is defined by "path 3" in section 4 of RFC 8989. It qualifies + a person who has been a front-page listed author or editor of at least two IETF- + stream RFCs within the last five years. An I-D in the RFC Editor queue that was + approved by the IESG is treated as an RFC, using the date of entry to the RFC + Editor queue as the date for qualification. + + This method does not strictly enforce "in the RFC Editor queue" for IESG-approved + drafts when computing eligibility. In the overwhelming majority of cases, an IESG- + approved draft immediately enters the queue and goes on to be published, so this + simplification makes the calculation much easier and virtually never affects + eligibility. + + Arguments eligibility_period_start and eligibility_period_end are datetimes that + mark the start and end of the eligibility period. These should be five years apart. + """ + # First, get the RFCs using publication date + qualifying_rfc_pub_events = DocEvent.objects.filter( + type='published_rfc', + time__gte=eligibility_period_start, + time__lte=eligibility_period_end, + ) + qualifying_rfcs = Document.objects.filter( + type_id="rfc", + docevent__in=qualifying_rfc_pub_events + ).annotate( + rfcauthor_count=Count("rfcauthor") + ) + rfcs_with_rfcauthors = qualifying_rfcs.filter(rfcauthor_count__gt=0).distinct() + rfcs_without_rfcauthors = qualifying_rfcs.filter(rfcauthor_count=0).distinct() + + # Second, get the IESG-approved I-Ds excluding any we're already counting as rfcs + qualifying_approval_events = DocEvent.objects.filter( + type='iesg_approved', + time__gte=eligibility_period_start, + time__lte=eligibility_period_end, + ) + qualifying_drafts = Document.objects.filter( + type_id="draft", + docevent__in=qualifying_approval_events, + ).exclude( + relateddocument__relationship_id="became_rfc", + relateddocument__target__in=qualifying_rfcs, + ).distinct() + + return base_qs.filter( + Q(documentauthor__document__in=qualifying_drafts) + | Q(rfcauthor__document__in=rfcs_with_rfcauthors) + | Q(documentauthor__document__in=rfcs_without_rfcauthors) + ).annotate( + document_author_count=Count('documentauthor'), + rfc_author_count=Count("rfcauthor") + ).annotate( + authorship_count=F("document_author_count") + F("rfc_author_count") + ).filter(authorship_count__gte=2) + + def get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable): if not base_qs: base_qs = Person.objects.all() @@ -608,14 +672,7 @@ def get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable): ) ).distinct() - rfc_pks = set(DocEvent.objects.filter(type='published_rfc', time__gte=five_years_ago, time__lte=date_as_dt).values_list('doc__pk', flat=True)) - iesgappr_pks = set(DocEvent.objects.filter(type='iesg_approved', time__gte=five_years_ago, time__lte=date_as_dt).values_list('doc__pk',flat=True)) - qualifying_pks = rfc_pks.union(iesgappr_pks.difference(rfc_pks)) - author_qs = base_qs.filter( - documentauthor__document__pk__in=qualifying_pks - ).annotate( - document_author_count = Count('documentauthor') - ).filter(document_author_count__gte=2) + author_qs = get_qualified_author_queryset(base_qs, five_years_ago, date_as_dt) return three_of_five_qs, officer_qs, author_qs def list_eligible_8989(date, base_qs=None): @@ -691,18 +748,42 @@ def three_of_five_eligible_9389(previous_five, queryset=None): counts[id] += 1 return queryset.filter(pk__in=[id for id, count in counts.items() if count >= 3]) -def suggest_affiliation(person): +def suggest_affiliation(person) -> str: + """Heuristically suggest a current affiliation for a Person""" recent_meeting = person.registration_set.order_by('-meeting__date').first() - affiliation = recent_meeting.affiliation if recent_meeting else '' - if not affiliation: - recent_volunteer = person.volunteer_set.order_by('-nomcom__group__acronym').first() - if recent_volunteer: - affiliation = recent_volunteer.affiliation - if not affiliation: - recent_draft_revision = NewRevisionDocEvent.objects.filter(doc__type_id='draft',doc__documentauthor__person=person).order_by('-time').first() - if recent_draft_revision: - affiliation = recent_draft_revision.doc.documentauthor_set.filter(person=person).first().affiliation - return affiliation + if recent_meeting and recent_meeting.affiliation: + return recent_meeting.affiliation + + recent_volunteer = person.volunteer_set.order_by('-nomcom__group__acronym').first() + if recent_volunteer and recent_volunteer.affiliation: + return recent_volunteer.affiliation + + recent_draft_revision = NewRevisionDocEvent.objects.filter( + doc__type_id="draft", + doc__documentauthor__person=person, + ).order_by("-time").first() + if recent_draft_revision: + draft_author = recent_draft_revision.doc.documentauthor_set.filter( + person=person + ).first() + if draft_author and draft_author.affiliation: + return draft_author.affiliation + + recent_rfc_publication = DocEvent.objects.filter( + Q(doc__documentauthor__person=person) | Q(doc__rfcauthor__person=person), + doc__type_id="rfc", + type="published_rfc", + ).order_by("-time").first() + if recent_rfc_publication: + rfc = recent_rfc_publication.doc + if rfc.rfcauthor_set.exists(): + rfc_author = rfc.rfcauthor_set.filter(person=person).first() + else: + rfc_author = rfc.documentauthor_set.filter(person=person).first() + if rfc_author and rfc_author.affiliation: + return rfc_author.affiliation + return "" + def extract_volunteers(year): nomcom = get_nomcom_by_year(year) diff --git a/ietf/person/models.py b/ietf/person/models.py index 03cf0c87fb..3ab89289a6 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -87,7 +87,7 @@ def short(self): else: prefix, first, middle, last, suffix = self.ascii_parts() return (first and first[0]+"." or "")+(middle or "")+" "+last+(suffix and " "+suffix or "") - def plain_name(self): + def plain_name(self) -> str: if not hasattr(self, '_cached_plain_name'): if self.plain: self._cached_plain_name = self.plain @@ -203,7 +203,10 @@ def has_drafts(self): def rfcs(self): from ietf.doc.models import Document - rfcs = list(Document.objects.filter(documentauthor__person=self, type='rfc')) + # When RfcAuthors are populated, this may over-return if an author is dropped + # from the author list between the final draft and the published RFC. Should + # ignore DocumentAuthors when an RfcAuthor exists for a draft. + rfcs = list(Document.objects.filter(type="rfc").filter(models.Q(documentauthor__person=self)|models.Q(rfcauthor__person=self)).distinct()) rfcs.sort(key=lambda d: d.name ) return rfcs @@ -266,11 +269,16 @@ def available_api_endpoints(self): def cdn_photo_url(self, size=80): if self.photo: if settings.SERVE_CDN_PHOTOS: + if settings.SERVER_MODE != "production": + original_media_dir = settings.MEDIA_URL + settings.MEDIA_URL = "https://www.ietf.org/lib/dt/media/" source_url = self.photo.url if source_url.startswith(settings.IETF_HOST_URL): source_url = source_url[len(settings.IETF_HOST_URL):] elif source_url.startswith('/'): source_url = source_url[1:] + if settings.SERVER_MODE != "production": + settings.MEDIA_URL = original_media_dir return f'{settings.IETF_HOST_URL}cdn-cgi/image/fit=scale-down,width={size},height={size}/{source_url}' else: datatracker_photo_path = urlreverse('ietf.person.views.photo', kwargs={'email_or_name': self.email()}) diff --git a/ietf/secr/telechat/tests.py b/ietf/secr/telechat/tests.py index fa26d33a5c..91ccde2187 100644 --- a/ietf/secr/telechat/tests.py +++ b/ietf/secr/telechat/tests.py @@ -256,7 +256,7 @@ def test_doc_detail_post_update_state_action_holder_automation(self): self.assertEqual(response.status_code,302) draft = Document.objects.get(name=draft.name) self.assertEqual(draft.get_state('draft-iesg').slug,'defer') - self.assertCountEqual(draft.action_holders.all(), [draft.ad] + draft.authors()) + self.assertCountEqual(draft.action_holders.all(), [draft.ad] + draft.author_persons()) self.assertEqual(draft.docevent_set.filter(type='changed_action_holders').count(), 1) # Removing need-rev should remove authors @@ -273,7 +273,7 @@ def test_doc_detail_post_update_state_action_holder_automation(self): # Setting to approved should remove all action holders # noinspection DjangoOrm - draft.action_holders.add(*(draft.authors())) # add() with through model ok in Django 2.2+ + draft.action_holders.add(*(draft.author_persons())) # add() with through model ok in Django 2.2+ response = self.client.post(url,{ 'submit': 'update_state', 'state': State.objects.get(type_id='draft-iesg', slug='approved').pk, diff --git a/ietf/settings.py b/ietf/settings.py index 05eab0f12f..fedd313ca0 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -22,6 +22,7 @@ warnings.filterwarnings("ignore", message="The django.utils.timezone.utc alias is deprecated.", module="oidc_provider") warnings.filterwarnings("ignore", message="The django.utils.datetime_safe module is deprecated.", module="tastypie") warnings.filterwarnings("ignore", message="The USE_DEPRECATED_PYTZ setting,") # https://github.com/ietf-tools/datatracker/issues/5635 +warnings.filterwarnings("ignore", message="The is_dst argument to make_aware\\(\\)") # caused by django-filters when USE_DEPRECATED_PYTZ is true warnings.filterwarnings("ignore", message="The USE_L10N setting is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5648 warnings.filterwarnings("ignore", message="django.contrib.auth.hashers.CryptPasswordHasher is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5663 @@ -502,6 +503,7 @@ def skip_unreadable_post(record): 'django_celery_results', 'corsheaders', 'django_markup', + 'django_filters', 'oidc_provider', 'drf_spectacular', 'drf_standardized_errors', diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 216fc7de6b..400d0d8c7d 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -595,7 +595,7 @@ def submit_existing(self, formats, change_authors=True, group_type='wg', stream_ TestBlobstoreManager().emptyTestBlobstores() def _assert_authors_are_action_holders(draft, expect=True): - for author in draft.authors(): + for author in draft.author_persons(): if expect: self.assertIn(author, draft.action_holders.all()) else: diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index a0c7dd8511..9a7c358a6d 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -1268,7 +1268,7 @@ def process_submission_text(filename, revision): if title: title = _normalize_title(title) - # Translation taable drops \r, \n, <, >. + # Translation table drops \r, \n, <, >. trans_table = str.maketrans("", "", "\r\n<>") authors = [ { diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py index b3234a87e2..cdcdeb5989 100644 --- a/ietf/sync/rfceditor.py +++ b/ietf/sync/rfceditor.py @@ -468,14 +468,18 @@ def update_docs_from_rfc_index( doc.set_state(rfc_published_state) if draft: doc.formal_languages.set(draft.formal_languages.all()) - for author in draft.documentauthor_set.all(): + # Create authors based on the last draft in the datatracker. This + # path will go away when we publish via the modernized RPC workflow + # but until then, these are the only data we have for authors that + # are easily connected to Person records. + for documentauthor in draft.documentauthor_set.all(): # Copy the author but point at the new doc. # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances - author.pk = None - author.id = None - author._state.adding = True - author.document = doc - author.save() + documentauthor.pk = None + documentauthor.id = None + documentauthor._state.adding = True + documentauthor.document = doc + documentauthor.save() if draft: draft_events = [] diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 3432f6214a..888920ae9d 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -446,7 +446,7 @@ def test_rfc_index(self): rfc_doc = Document.objects.filter(rfc_number=1234, type_id="rfc").first() self.assertIsNotNone(rfc_doc, "RFC document should have been created") - self.assertEqual(rfc_doc.authors(), draft_doc.authors()) + self.assertEqual(rfc_doc.author_persons_or_names(), draft_doc.author_persons_or_names()) rfc_events = rfc_doc.docevent_set.all() self.assertEqual(len(rfc_events), 8) expected_events = [ diff --git a/ietf/templates/doc/document_info.html b/ietf/templates/doc/document_info.html index 71050f9d41..d6d8d43071 100644 --- a/ietf/templates/doc/document_info.html +++ b/ietf/templates/doc/document_info.html @@ -87,7 +87,7 @@ {% endif %}
        Author{% if doc.pk %}{{ doc.authors|pluralize }}{% endif %}Author{% if doc.pk %}{{ doc.author_persons_or_names|pluralize }}{% endif %} {% if can_edit_authors %} {# Implementation that uses the current primary email for each author #} - {% if doc.pk %}{% for author in doc.authors %} - {% person_link author %}{% if not forloop.last %},{% endif %} + {% if doc.pk %}{% for author in doc.author_persons_or_names %} + {% if author.person %}{% person_link author.person %}{% else %}{{ author.titlepage_name }}{% endif %}{% if not forloop.last %},{% endif %} {% endfor %}{% endif %} {% if document_html and not snapshot or document_html and doc.rev == latest_rev%}
        diff --git a/ietf/templates/doc/index_active_drafts.html b/ietf/templates/doc/index_active_drafts.html index 06ea2c4ff5..607385f56f 100644 --- a/ietf/templates/doc/index_active_drafts.html +++ b/ietf/templates/doc/index_active_drafts.html @@ -29,7 +29,7 @@

        Active Internet-Drafts

        {% for group in groups %}

        {{ group.name }} ({{ group.acronym }})

        - {% for d in group.active_drafts %} + {% for d in group.active_drafts %}{# n.b., d is a dict, not a Document #}
        {{ d.title }}. diff --git a/ietf/templates/doc/opengraph.html b/ietf/templates/doc/opengraph.html index 4fe39b6209..1c8c5abe91 100644 --- a/ietf/templates/doc/opengraph.html +++ b/ietf/templates/doc/opengraph.html @@ -1,4 +1,4 @@ -{# Copyright The IETF Trust 2016-2020, All Rights Reserved #} +{# Copyright The IETF Trust 2016-2025, All Rights Reserved #} {% load origin %} {% load static %} {% load ietf_filters %} @@ -36,7 +36,7 @@ {% else %}{# TODO: We need a card image for individual I-Ds. #} {% endif %} -{% if doc.pk %}{% for author in doc.documentauthor_set.all %} +{% if doc.pk %}{% for author_name in doc.author_names %} {% endfor %}{% endif %} {% if published %}{% endif %} {% if expires %}{% endif %} \ No newline at end of file diff --git a/ietf/templates/doc/review/request_info.html b/ietf/templates/doc/review/request_info.html index 9ad126d59e..51aea10a02 100644 --- a/ietf/templates/doc/review/request_info.html +++ b/ietf/templates/doc/review/request_info.html @@ -74,13 +74,13 @@
        		{% person_link review_req.requested_by %}
        Authors - {% for author in review_req.doc.authors %} - {% person_link author %}{% if not forloop.last %},{% endif %} + {% for person, tp_name in review_req.doc.author_persons_or_names %} + {% if person %}{% person_link person %}{% else %}{{ tp_name }}{% endif %}{% if not forloop.last %},{% endif %} {% endfor %}
        @@ -44,4 +89,8 @@

        Attendee list of IETF {{ meeting.number }} meeting

        {% endblock %} {% block js %} -{% endblock %} \ No newline at end of file + {% if chart_data %} + + + {% endif %} +{% endblock %} diff --git a/package.json b/package.json index fec29275b4..bb71250c4b 100644 --- a/package.json +++ b/package.json @@ -112,6 +112,7 @@ "ietf/static/images/irtf-logo-white.svg", "ietf/static/images/irtf-logo.svg", "ietf/static/js/add_session_recordings.js", + "ietf/static/js/attendees-chart.js", "ietf/static/js/agenda_filter.js", "ietf/static/js/agenda_materials.js", "ietf/static/js/announcement.js", From 2c29cbaad91a7c076643167e1ffc056975b2c97e Mon Sep 17 00:00:00 2001 From: Tianyi Gao Date: Fri, 20 Mar 2026 11:45:09 +0800 Subject: [PATCH 092/158] feat: add parent section in team about (#9148) (#10551) fix: remove empty for area/parent on all groups --- ietf/group/tests_info.py | 19 +++++++++++++++++++ ietf/templates/group/group_about.html | 11 +++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 34f8500854..3f24e2e3d6 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -543,6 +543,25 @@ def verify_can_edit_group(url, group, username): for username in list(set(interesting_users)-set(can_edit[group.type_id])): verify_cannot_edit_group(url, group, username) + def test_group_about_team_parent(self): + """Team about page should show parent when parent is not an area""" + GroupFactory(type_id='team', parent=GroupFactory(type_id='area', acronym='gen')) + GroupFactory(type_id='team', parent=GroupFactory(type_id='ietf', acronym='iab')) + GroupFactory(type_id='team', parent=None) + + for team in Group.objects.filter(type='team').select_related('parent'): + url = urlreverse('ietf.group.views.group_about', kwargs=dict(acronym=team.acronym)) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + if team.parent and team.parent.type_id != 'area': + self.assertContains(r, 'Parent') + self.assertContains(r, team.parent.acronym) + elif team.parent and team.parent.type_id == 'area': + self.assertContains(r, team.parent.name) + self.assertNotContains(r, '>Parent<') + else: + self.assertNotContains(r, '>Parent<') + def test_group_about_personnel(self): """Correct personnel should appear on the group About page""" group = GroupFactory() diff --git a/ietf/templates/group/group_about.html b/ietf/templates/group/group_about.html index cbc2e11536..6d1843383c 100644 --- a/ietf/templates/group/group_about.html +++ b/ietf/templates/group/group_about.html @@ -51,10 +51,13 @@ {{ group.parent.name }} ({{ group.parent.acronym }}) - {% else %} - + {% elif group.parent and group.type_id == "team" %} + - + {% endif %} @@ -444,4 +447,4 @@

        group_stats("{% url 'ietf.group.views.group_stats_data' %}", ".chart"); }); -{% endblock %} \ No newline at end of file +{% endblock %} From abab6373f5f465bdcc052f45c5def0710f360dc7 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 19 Mar 2026 23:02:40 -0500 Subject: [PATCH 093/158] fix: reduce db churn and log noise for rebuilding references (#10563) * fix: (wip) reduce db churn and log noise for rebuilding references * fix: typo in log message * fix: typo in log message Co-authored-by: Jennifer Richards --------- Co-authored-by: Jennifer Richards --- ietf/doc/utils.py | 78 ++++++++++++++++++++++++++++++-------------- ietf/submit/utils.py | 5 +-- 2 files changed, 54 insertions(+), 29 deletions(-) diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 396b3fcfa4..8cbe5e8f3e 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -4,6 +4,7 @@ import datetime import io +import json import math import os import re @@ -954,58 +955,78 @@ def rebuild_reference_relations(doc, filenames): filenames should be a dict mapping file ext (i.e., type) to the full path of each file. """ if doc.type.slug not in ["draft", "rfc"]: + log.log(f"rebuild_reference_relations called for non draft/rfc doc {doc.name}") return None - - log.log(f"Rebuilding reference relations for {doc.name}") - # try XML first - if "xml" in filenames: - refs = XMLDraft(filenames["xml"]).get_refs() - elif "txt" in filenames: - filename = filenames["txt"] - try: - refs = draft.PlaintextDraft.from_file(filename).get_refs() - except IOError as e: - return {"errors": [f"{e.strerror}: {filename}"]} - else: + + if "xml" not in filenames and "txt" not in filenames: + log.log(f"rebuild_reference_relations error: no file available for {doc.name}") return { "errors": [ "No file available for rebuilding reference relations. Need XML or plaintext." ] } - - doc.relateddocument_set.filter( + else: + try: + # try XML first + if "xml" in filenames: + refs = XMLDraft(filenames["xml"]).get_refs() + elif "txt" in filenames: + filename = filenames["txt"] + refs = draft.PlaintextDraft.from_file(filename).get_refs() + except (IOError, UnicodeDecodeError) as e: + log.log(f"rebuild_reference_relations error: On {doc.name}: {e}") + return {"errors": [f"{e}: {filename}"]} + + before = set(doc.relateddocument_set.filter( relationship__slug__in=["refnorm", "refinfo", "refold", "refunk"] - ).delete() + ).values_list("relationship__slug","target__name")) warnings = [] errors = [] unfound = set() + intended = set() + names = [ref for ref in refs] + names.extend([ref[:-3] for ref in refs if re.match(r"^draft-.*-\d{2}$", ref)]) + queryset = Document.objects.filter(name__in=names) for ref, refType in refs.items(): - refdoc = Document.objects.filter(name=ref) - if not refdoc and re.match(r"^draft-.*-\d{2}$", ref): - refdoc = Document.objects.filter(name=ref[:-3]) + refdoc = queryset.filter(name=ref) + if not refdoc.exists() and re.match(r"^draft-.*-\d{2}$", ref): + refdoc = queryset.filter(name=ref[:-3]) count = refdoc.count() if count == 0: unfound.add("%s" % ref) continue elif count > 1: + log.unreachable("2026-3-16") # This branch is holdover from DocAlias errors.append("Too many Document objects found for %s" % ref) else: # Don't add references to ourself if doc != refdoc[0]: - RelatedDocument.objects.get_or_create( - source=doc, - target=refdoc[0], - relationship=DocRelationshipName.objects.get( - slug="ref%s" % refType - ), - ) + intended.add((f"ref{refType}", refdoc[0].name)) + if unfound: warnings.append( "There were %d references with no matching Document" % len(unfound) ) + if intended != before: + for slug, name in before-intended: + doc.relateddocument_set.filter(target__name=name, relationship_id=slug).delete() + for slug, name in intended-before: + doc.relateddocument_set.create( + target=queryset.get(name=name), + relationship_id=slug + ) + after = set(doc.relateddocument_set.filter( + relationship__slug__in=["refnorm", "refinfo", "refold", "refunk"] + ).values_list("relationship__slug","target__name")) + if after != intended: + errors.append("Attempted changed didn't achieve intended results") + changed_references = True + else: + changed_references = False + ret = {} if errors: ret["errors"] = errors @@ -1014,6 +1035,13 @@ def rebuild_reference_relations(doc, filenames): if unfound: ret["unfound"] = list(unfound) + logmsg = f"rebuild_reference_relations for {doc.name}: " + logmsg += "changed references" if changed_references else "references unchanged" + if ret: + logmsg += f" {json.dumps(ret)}" + + log.log(logmsg) + return ret def set_replaces_for_document(request, doc, new_replaces, by, email_subject, comment=""): diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 9a7c358a6d..7e3106f723 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -395,10 +395,7 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc): log.log(f"{submission.name}: updated state and info") - trouble = rebuild_reference_relations(draft, find_submission_filenames(draft)) - if trouble: - log.log('Rebuild_reference_relations trouble: %s'%trouble) - log.log(f"{submission.name}: rebuilt reference relations") + rebuild_reference_relations(draft, find_submission_filenames(draft)) if draft.stream_id == "ietf" and draft.group.type_id == "wg" and draft.rev == "00": # automatically set state "WG Document" From b08945aaf4618613f668bb5c231533f709bea4d4 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 20 Mar 2026 05:17:10 -0300 Subject: [PATCH 094/158] fix: maintain column count in HTML template (#10593) --- ietf/templates/group/group_about.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ietf/templates/group/group_about.html b/ietf/templates/group/group_about.html index 6d1843383c..0a8b9194f2 100644 --- a/ietf/templates/group/group_about.html +++ b/ietf/templates/group/group_about.html @@ -58,6 +58,10 @@ {{ group.parent.name }} ({{ group.parent.acronym }}) + {% else %} +

        + + {% endif %} From d39317b070a7af5db4f48edaf0e7f03fd0a29680 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 23 Mar 2026 12:33:23 -0300 Subject: [PATCH 095/158] feat: update typesense search index on rfc pub/update (#10575) * chore: typesense API config for k8s * feat: DocumentInfo.pub_datetime() helper * chore(deps): install typesense library * feat: searchindex (typesense) util module * feat: sanitize abstract * feat: add (sanitized) content * style: ruff ruff on doc/tasks.py * feat: search index update task * chore: call the update task * refactor: better settings management * ci: update prod settings * chore: typing * test: searchindex tests * test: searchindex task test * style: ruff ruff * chore: drop type hints to fix mypy errors * test: fix tests * test: improve coverage * fix: handle missing content blob correctly --- ietf/api/serializers_rpc.py | 5 +- ietf/api/tests_serializers_rpc.py | 18 +++- ietf/api/tests_views_rpc.py | 44 ++++++--- ietf/api/views_rpc.py | 3 +- ietf/doc/models.py | 17 ++-- ietf/doc/tasks.py | 46 +++++++-- ietf/doc/tests_tasks.py | 64 ++++++++++-- ietf/utils/searchindex.py | 155 ++++++++++++++++++++++++++++++ ietf/utils/tests_searchindex.py | 128 ++++++++++++++++++++++++ k8s/settings_local.py | 20 ++-- requirements.txt | 1 + 11 files changed, 451 insertions(+), 50 deletions(-) create mode 100644 ietf/utils/searchindex.py create mode 100644 ietf/utils/tests_searchindex.py diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index a18dc588c4..701f05eece 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2025, All Rights Reserved +# Copyright The IETF Trust 2025-2026, All Rights Reserved import datetime from pathlib import Path from typing import Literal, Optional @@ -20,6 +20,7 @@ RfcAuthor, ) from ietf.doc.serializers import RfcAuthorSerializer +from ietf.doc.tasks import update_rfc_searchindex_task from ietf.doc.utils import ( default_consensus, prettify_std_name, @@ -682,6 +683,8 @@ def update(self, instance, validated_data): stale_subseries_relations.delete() if len(rfc_events) > 0: rfc.save_with_history(rfc_events) + + update_rfc_searchindex_task.delay(rfc.rfc_number) return rfc diff --git a/ietf/api/tests_serializers_rpc.py b/ietf/api/tests_serializers_rpc.py index 1babb4c30f..ed326be451 100644 --- a/ietf/api/tests_serializers_rpc.py +++ b/ietf/api/tests_serializers_rpc.py @@ -1,4 +1,6 @@ # Copyright The IETF Trust 2026, All Rights Reserved +from unittest import mock + from django.utils import timezone from ietf.utils.test_utils import TestCase @@ -32,7 +34,8 @@ def test_create(self): with self.assertRaises(RuntimeError, msg="serializer does not allow create()"): serializer.save() - def test_update(self): + @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") + def test_update(self, mock_update_searchindex_task): rfc = WgRfcFactory(pages=10) serializer = EditableRfcSerializer( instance=rfc, @@ -56,6 +59,11 @@ def test_update(self): ) self.assertTrue(serializer.is_valid()) result = serializer.save() + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) result.refresh_from_db() self.assertEqual(result.title, "Yadda yadda yadda") self.assertEqual( @@ -84,7 +92,8 @@ def test_update(self): [Document.objects.get(name="fyi999")], ) - def test_partial_update(self): + @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") + def test_partial_update(self, mock_update_searchindex_task): # We could test other permutations of fields, but authors is a partial update # we know we are going to use, so verifying that one in particular. rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan") @@ -104,6 +113,11 @@ def test_partial_update(self): ) self.assertTrue(serializer.is_valid()) result = serializer.save() + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) result.refresh_from_db() self.assertEqual(rfc.title, "padawan") self.assertEqual( diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 7ab8778d28..a679e74789 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -196,7 +196,8 @@ def test_notify_rfc_published(self, mock_task_delay): self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) - def test_upload_rfc_files(self): + @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task") + def test_upload_rfc_files(self, mock_update_searchindex_task): def _valid_post_data(): """Generate a valid post data dict @@ -217,14 +218,7 @@ def _valid_post_data(): } url = urlreverse("ietf.api.purple_api.upload_rfc_files") - unused_rfc_number = ( - Document.objects.filter(rfc_number__isnull=False).aggregate( - unused_rfc_number=Max("rfc_number") + 1 - )["unused_rfc_number"] - or 10000 - ) - - rfc = WgRfcFactory(rfc_number=unused_rfc_number) + rfc = WgRfcFactory() assert isinstance(rfc, Document), "WgRfcFactory should generate a Document" with TemporaryDirectory() as rfc_dir: settings.RFC_PATH = rfc_dir # affects overridden settings @@ -236,15 +230,17 @@ def _valid_post_data(): # no api key r = self.client.post(url, _valid_post_data(), format="multipart") self.assertEqual(r.status_code, 403) + self.assertFalse(mock_update_searchindex_task.delay.called) # invalid RFC r = self.client.post( url, - _valid_post_data() | {"rfc": unused_rfc_number + 1}, + _valid_post_data() | {"rfc": rfc.rfc_number + 10}, format="multipart", headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) # empty files r = self.client.post( @@ -263,6 +259,7 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) # bad file type r = self.client.post( @@ -276,9 +273,10 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) # Put a file in the way. Post should fail because replace = False - file_in_the_way = (rfc_path / f"rfc{unused_rfc_number}.txt") + file_in_the_way = (rfc_path / f"{rfc.name}.txt") file_in_the_way.touch() r = self.client.post( url, @@ -287,11 +285,12 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) file_in_the_way.unlink() # Put a blob in the way. Post should fail because replace = False blob_in_the_way = Blob.objects.create( - bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b"" + bucket="rfc", name=f"txt/{rfc.name}.txt", content=b"" ) r = self.client.post( url, @@ -300,6 +299,7 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) blob_in_the_way.delete() # valid post @@ -310,8 +310,13 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 200) + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) for extension in ["xml", "txt", "html", "pdf", "json"]: - filename = f"rfc{unused_rfc_number}.{extension}" + filename = f"{rfc.name}.{extension}" self.assertEqual( (rfc_path / filename) .read_text(), @@ -328,7 +333,7 @@ def _valid_post_data(): f"{extension} blob should contain the expected content", ) # special case for notprepped - notprepped_fn = f"rfc{unused_rfc_number}.notprepped.xml" + notprepped_fn = f"{rfc.name}.notprepped.xml" self.assertEqual( ( rfc_path / "prerelease" / notprepped_fn @@ -347,6 +352,7 @@ def _valid_post_data(): ) # re-post with replace = False should now fail + mock_update_searchindex_task.reset_mock() r = self.client.post( url, _valid_post_data(), @@ -354,7 +360,8 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 409) # conflict - + self.assertFalse(mock_update_searchindex_task.delay.called) + # re-post with replace = True should succeed r = self.client.post( url, @@ -362,7 +369,12 @@ def _valid_post_data(): format="multipart", headers={"X-Api-Key": "valid-token"}, ) - self.assertEqual(r.status_code, 200) # conflict + self.assertEqual(r.status_code, 200) + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) @mock.patch("ietf.api.views_rpc.create_rfc_index_task") diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index c7ae699005..cb6a59a167 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -38,7 +38,7 @@ from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent from ietf.doc.serializers import RfcAuthorSerializer from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage -from ietf.doc.tasks import signal_update_rfc_metadata_task +from ietf.doc.tasks import signal_update_rfc_metadata_task, update_rfc_searchindex_task from ietf.person.models import Email, Person from ietf.sync.tasks import create_rfc_index_task @@ -516,6 +516,7 @@ def post(self, request): destination.parent.mkdir() shutil.move(ftm, destination) + update_rfc_searchindex_task.delay(rfc.rfc_number) return Response(NotificationAckSerializer().data) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 7b23a62c45..972f0a34e8 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1285,11 +1285,8 @@ def submission(self): s = s.first() return s - def pub_date(self): - """Get the publication date for this document - - This is the rfc publication date for RFCs, and the new-revision date for other documents. - """ + def pub_datetime(self): + """Get the publication datetime of this document""" if self.type_id == "rfc": # As of Sept 2022, in ietf.sync.rfceditor.update_docs_from_rfc_index() `published_rfc` events are # created with a timestamp whose date *in the PST8PDT timezone* is the official publication date @@ -1297,7 +1294,15 @@ def pub_date(self): event = self.latest_event(type='published_rfc') else: event = self.latest_event(type='new_revision') - return event.time.astimezone(RPC_TZINFO).date() if event else None + return event.time.astimezone(RPC_TZINFO) if event else None + + def pub_date(self): + """Get the publication date for this document + + This is the rfc publication date for RFCs, and the new-revision date for other documents. + """ + pub_datetime = self.pub_datetime() + return None if pub_datetime is None else pub_datetime.date() def is_dochistory(self): return False diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index 90f4c80af5..a38cd5eb5c 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -3,6 +3,7 @@ # Celery task definitions # import datetime + import debug # pyflakes:ignore from celery import shared_task @@ -11,7 +12,7 @@ from django.conf import settings from django.utils import timezone -from ietf.utils import log +from ietf.utils import log, searchindex from ietf.utils.timezone import datetime_today from .expire import ( @@ -77,17 +78,19 @@ def expire_last_calls_task(): try: expire_last_call(doc) except Exception: - log.log(f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})") + log.log( + f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})" + ) else: log.log(f"Expired last call for {doc.file_tag()} (id={doc.pk})") -@shared_task +@shared_task def generate_idnits2_rfc_status_task(): outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status" blob = generate_idnits2_rfc_status() try: - outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE + outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE except Exception as e: log.log(f"failed to write idnits2-rfc-status: {e}") @@ -97,7 +100,7 @@ def generate_idnits2_rfcs_obsoleted_task(): outpath = Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted" blob = generate_idnits2_rfcs_obsoleted() try: - outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE + outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE except Exception as e: log.log(f"failed to write idnits2-rfcs-obsoleted: {e}") @@ -105,7 +108,7 @@ def generate_idnits2_rfcs_obsoleted_task(): @shared_task def generate_draft_bibxml_files_task(days=7, process_all=False): """Generate bibxml files for recently updated docs - + If process_all is False (the default), processes only docs with new revisions in the last specified number of days. """ @@ -117,7 +120,9 @@ def generate_draft_bibxml_files_task(days=7, process_all=False): doc__type_id="draft", ).order_by("time") if not process_all: - doc_events = doc_events.filter(time__gte=timezone.now() - datetime.timedelta(days=days)) + doc_events = doc_events.filter( + time__gte=timezone.now() - datetime.timedelta(days=days) + ) for event in doc_events: try: update_or_create_draft_bibxml_file(event.doc, event.rev) @@ -132,6 +137,7 @@ def investigate_fragment_task(name_fragment: str): "results": investigate_fragment(name_fragment), } + @shared_task def rebuild_reference_relations_task(doc_names: list[str]): log.log(f"Task: Rebuilding reference relations for {doc_names}") @@ -157,6 +163,32 @@ def rebuild_reference_relations_task(doc_names: list[str]): def fixup_bofreq_timestamps_task(): # pragma: nocover fixup_bofreq_timestamps() + @shared_task def signal_update_rfc_metadata_task(rfc_number_list=()): signal_update_rfc_metadata(rfc_number_list) + + +@shared_task(bind=True) +def update_rfc_searchindex_task(self, rfc_number: int): + """Update the search index for one RFC""" + if not searchindex.enabled(): + log.log("Search indexing is not enabled, skipping") + return + + rfc = Document.objects.filter(type_id="rfc", rfc_number=rfc_number).first() + if rfc is None: + log.log( + f"ERROR: Document for rfc{rfc_number} not found, not updating search index" + ) + return + try: + searchindex.update_or_create_rfc_entry(rfc) + except Exception as err: + log.log(f"Search index update for {rfc.name} failed ({err})") + if isinstance(err, searchindex.RETRYABLE_ERROR_CLASSES): + searchindex_settings = searchindex.get_settings() + self.retry( + countdown=searchindex_settings["TASK_RETRY_DELAY"], + max_retries=searchindex_settings["TASK_MAX_RETRIES"], + ) diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 29689cd596..728d21f131 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -1,18 +1,20 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved -import debug # pyflakes:ignore import datetime from unittest import mock from pathlib import Path +from celery.exceptions import Retry from django.conf import settings +from django.test.utils import override_settings from django.utils import timezone +from typesense import exceptions as typesense_exceptions from ietf.utils.test_utils import TestCase from ietf.utils.timezone import datetime_today -from .factories import DocumentFactory, NewRevisionDocEventFactory +from .factories import DocumentFactory, NewRevisionDocEventFactory, WgRfcFactory from .models import Document, NewRevisionDocEvent from .tasks import ( expire_ids_task, @@ -22,8 +24,10 @@ generate_idnits2_rfc_status_task, investigate_fragment_task, notify_expirations_task, + update_rfc_searchindex_task, ) + class TaskTests(TestCase): @mock.patch("ietf.doc.tasks.in_draft_expire_freeze") @mock.patch("ietf.doc.tasks.get_expired_drafts") @@ -87,7 +91,7 @@ def test_expire_last_calls_task(self, mock_get_expired, mock_expire): self.assertEqual(mock_expire.call_args_list[0], mock.call(docs[0])) self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1])) self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2])) - + # Check that it runs even if exceptions occur mock_get_expired.reset_mock() mock_expire.reset_mock() @@ -111,9 +115,40 @@ def test_investigate_fragment_task(self): retval, {"name_fragment": "some fragment", "results": investigation_results} ) + @mock.patch("ietf.doc.tasks.searchindex.update_or_create_rfc_entry") + @mock.patch("ietf.doc.tasks.searchindex.enabled") + def test_update_rfc_searchindex_task( + self, mock_searchindex_enabled, mock_create_entry + ): + mock_searchindex_enabled.return_value = False + + self.assertFalse(Document.objects.filter(rfc_number=5073).exists()) + rfc = WgRfcFactory() + update_rfc_searchindex_task(rfc_number=5073) + self.assertFalse(mock_create_entry.called) + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + self.assertFalse(mock_create_entry.called) + + mock_searchindex_enabled.return_value = True + update_rfc_searchindex_task(rfc_number=5073) + self.assertFalse(mock_create_entry.called) + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + self.assertTrue(mock_create_entry.called) + + with override_settings(SEARCHINDEX_CONFIG={"TASK_MAX_RETRIES": 0}): + # Try a non-retryable error (there are others) + mock_create_entry.side_effect = typesense_exceptions.RequestMalformed + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) # no retry + # Now what should be a retryable error + mock_create_entry.side_effect = typesense_exceptions.Timeout + with self.assertRaises(Retry): + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + class Idnits2SupportTests(TestCase): - settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR'] + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + [ + "DERIVED_DIR" + ] @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted") def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate): @@ -151,7 +186,9 @@ def setUp(self): ) # a couple that should always be ignored NewRevisionDocEventFactory( - time=now - datetime.timedelta(days=6), rev="09", doc__type_id="rfc" # not a draft + time=now - datetime.timedelta(days=6), + rev="09", + doc__type_id="rfc", # not a draft ) NewRevisionDocEventFactory( type="changed_document", # not a "new_revision" type @@ -164,7 +201,9 @@ def setUp(self): @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_bibxml_files_for_all_drafts_task(self, mock_create, mock_ensure_path): + def test_generate_bibxml_files_for_all_drafts_task( + self, mock_create, mock_ensure_path + ): generate_draft_bibxml_files_task(process_all=True) self.assertTrue(mock_ensure_path.called) self.assertCountEqual( @@ -193,12 +232,15 @@ def test_generate_bibxml_files_for_all_drafts_task(self, mock_create, mock_ensur @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_ensure_path): + def test_generate_bibxml_files_for_recent_drafts_task( + self, mock_create, mock_ensure_path + ): # default args - look back 7 days generate_draft_bibxml_files_task() self.assertTrue(mock_ensure_path.called) self.assertCountEqual( - mock_create.call_args_list, [mock.call(self.young_event.doc, self.young_event.rev)] + mock_create.call_args_list, + [mock.call(self.young_event.doc, self.young_event.rev)], ) mock_create.reset_mock() mock_ensure_path.reset_mock() @@ -223,7 +265,9 @@ def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_en @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value(self, mock_create, mock_ensure_path): + def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value( + self, mock_create, mock_ensure_path + ): with self.assertRaises(ValueError): generate_draft_bibxml_files_task(days=0) self.assertFalse(mock_create.called) diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py new file mode 100644 index 0000000000..e4427b88b5 --- /dev/null +++ b/ietf/utils/searchindex.py @@ -0,0 +1,155 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +"""Search indexing utilities""" + +import re +from math import floor + +import httpx # just for exceptions +import typesense +import typesense.exceptions +from django.conf import settings + +from ietf.doc.models import Document, StoredObject +from ietf.doc.storage_utils import retrieve_str +from ietf.utils.log import log + +# Error classes that might succeed just by retrying a failed attempt. +# Must be a tuple for use with isinstance() +RETRYABLE_ERROR_CLASSES = ( + httpx.ConnectError, + httpx.ConnectTimeout, + typesense.exceptions.Timeout, + typesense.exceptions.ServerError, + typesense.exceptions.ServiceUnavailable, +) + + +DEFAULT_SETTINGS = { + "TYPESENSE_API_URL": "", + "TYPESENSE_API_KEY": "", + "TYPESENSE_COLLECTION_NAME": "docs", + "TASK_RETRY_DELAY": 10, + "TASK_MAX_RETRIES": 12, +} + + +def get_settings(): + return DEFAULT_SETTINGS | getattr(settings, "SEARCHINDEX_CONFIG", {}) + + +def enabled(): + _settings = get_settings() + return _settings["TYPESENSE_API_URL"] != "" + + +def _sanitize_text(content): + """Sanitize content or abstract text for search""" + # REs (with approximate names) + RE_DOT_OR_BANG_SPACE = r"\. |! " # -> " " (space) + RE_COMMENT_OR_TOC_CRUD = r"<--|-->|--+|\+|\.\.+" # -> "" + RE_BRACKETED_REF = r"\[[a-zA-Z0-9 -]+\]" # -> "" + RE_DOTTED_NUMBERS = r"[0-9]+\.[0-9]+(\.[0-9]+)?" # -> "" + RE_MULTIPLE_WHITESPACE = r"\s+" # -> " " (space) + # Replacement values (for clarity of intent) + SPACE = " " + EMPTY = "" + # Sanitizing begins here, order is significant! + content = re.sub(RE_DOT_OR_BANG_SPACE, SPACE, content.strip()) + content = re.sub(RE_COMMENT_OR_TOC_CRUD, EMPTY, content) + content = re.sub(RE_BRACKETED_REF, EMPTY, content) + content = re.sub(RE_DOTTED_NUMBERS, EMPTY, content) + content = re.sub(RE_MULTIPLE_WHITESPACE, SPACE, content) + return content.strip() + + +def update_or_create_rfc_entry(rfc: Document): + assert rfc.type_id == "rfc" + assert rfc.rfc_number is not None + + keywords: list[str] = rfc.keywords # help type checking + + subseries = rfc.part_of() + if len(subseries) > 1: + log( + f"RFC {rfc.rfc_number} is in multiple subseries. " + f"Indexing as {subseries[0].name}" + ) + subseries = subseries[0] if len(subseries) > 0 else None + obsoleted_by = rfc.relations_that("obs") + updated_by = rfc.relations_that("updates") + + stored_txt = ( + StoredObject.objects.exclude_deleted() + .filter(store="rfc", doc_name=rfc.name, name__startswith="txt/") + .first() + ) + content = "" + if stored_txt is not None: + # Should be available in the blobdb, but be cautious... + try: + content = retrieve_str(kind=stored_txt.store, name=stored_txt.name) + except Exception as err: + log(f"Unable to retrieve {stored_txt} from storage: {err}") + + ts_id = f"doc-{rfc.pk}" + ts_document = { + "rfcNumber": rfc.rfc_number, + "rfc": str(rfc.rfc_number), + "filename": rfc.name, + "title": rfc.title, + "abstract": _sanitize_text(rfc.abstract), + "keywords": keywords, + "type": "rfc", + "state": [state.name for state in rfc.states.all()], + "status": {"slug": rfc.std_level.slug, "name": rfc.std_level.name}, + "date": floor(rfc.time.timestamp()), + "publicationDate": floor(rfc.pub_datetime().timestamp()), + "stream": {"slug": rfc.stream.slug, "name": rfc.stream.name}, + "authors": [ + {"name": rfc_author.titlepage_name, "affiliation": rfc_author.affiliation} + for rfc_author in rfc.rfcauthor_set.all() + ], + "flags": { + "hiddenDefault": False, + "obsoleted": len(obsoleted_by) > 0, + "updated": len(updated_by) > 0, + }, + "obsoletedBy": [str(doc.rfc_number) for doc in obsoleted_by], + "updatedBy": [str(doc.rfc_number) for doc in updated_by], + "ranking": rfc.rfc_number, + } + if subseries is not None: + ts_document["subseries"] = { + "acronym": subseries.type.slug, + "number": int(subseries.name[len(subseries.type.slug) :]), + "total": len(subseries.contains()), + } + if rfc.group is not None: + ts_document["group"] = { + "acronym": rfc.group.acronym, + "name": rfc.group.name, + "full": f"{rfc.group.acronym} - {rfc.group.name}", + } + if ( + rfc.group.parent is not None + and rfc.stream_id not in ["ise", "irtf", "iab"] # exclude editorial? + ): + ts_document["area"] = { + "acronym": rfc.group.parent.acronym, + "name": rfc.group.parent.name, + "full": f"{rfc.group.parent.acronym} - {rfc.group.parent.name}", + } + if rfc.ad is not None: + ts_document["adName"] = rfc.ad.name + if content != "": + ts_document["content"] = _sanitize_text(content) + _settings = get_settings() + client = typesense.Client( + { + "api_key": _settings["TYPESENSE_API_KEY"], + "nodes": [_settings["TYPESENSE_API_URL"]], + } + ) + client.collections[_settings["TYPESENSE_COLLECTION_NAME"]].documents.upsert( + {"id": ts_id} | ts_document + ) diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py new file mode 100644 index 0000000000..8740716c85 --- /dev/null +++ b/ietf/utils/tests_searchindex.py @@ -0,0 +1,128 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from unittest import mock + +from django.conf import settings +from django.test.utils import override_settings + +from . import searchindex +from .test_utils import TestCase +from ..blobdb.models import Blob +from ..doc.factories import ( + WgDraftFactory, + WgRfcFactory, + PublishedRfcDocEventFactory, + BcpFactory, + StdFactory, +) +from ..doc.models import Document +from ..doc.storage_utils import store_str +from ..person.factories import PersonFactory + + +class SearchindexTests(TestCase): + def test_enabled(self): + with override_settings(): + try: + del settings.SEARCHINDEX_CONFIG + except AttributeError: + pass + self.assertFalse(searchindex.enabled()) + with override_settings( + SEARCHINDEX_CONFIG={"TYPESENSE_API_KEY": "this-is-not-a-key"} + ): + self.assertFalse(searchindex.enabled()) + with override_settings( + SEARCHINDEX_CONFIG={"TYPESENSE_API_URL": "http://example.com"} + ): + self.assertTrue(searchindex.enabled()) + + def test_sanitize_text(self): + dirty_text = """ + + This is text. It + is <---- full of \tprobl.....ems! Fix it. + """ + sanitized = "This is text It is full of problems Fix it." + self.assertEqual(searchindex._sanitize_text(dirty_text), sanitized) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_update_or_create_rfc_entry(self, mock_ts_client_constructor): + not_rfc = WgDraftFactory() + assert isinstance(not_rfc, Document) + with self.assertRaises(AssertionError): + searchindex.update_or_create_rfc_entry(not_rfc) + self.assertFalse(mock_ts_client_constructor.called) + + invalid_rfc = WgRfcFactory(name="rfc1000000", rfc_number=None) + assert isinstance(invalid_rfc, Document) + with self.assertRaises(AssertionError): + searchindex.update_or_create_rfc_entry(invalid_rfc) + self.assertFalse(mock_ts_client_constructor.called) + + rfc = PublishedRfcDocEventFactory().doc + assert isinstance(rfc, Document) + searchindex.update_or_create_rfc_entry(rfc) + self.assertTrue(mock_ts_client_constructor.called) + # walk the tree down to the method we expected to be called... + mock_upsert = mock_ts_client_constructor.return_value.collections[ + "frogs" + ].documents.upsert # matches value in override_settings above + self.assertTrue(mock_upsert.called) + upserted_dict = mock_upsert.call_args[0][0] + # Check a few values, not exhaustive + self.assertEqual(upserted_dict["id"], f"doc-{rfc.pk}") + self.assertEqual(upserted_dict["rfcNumber"], rfc.rfc_number) + self.assertEqual( + upserted_dict["abstract"], searchindex._sanitize_text(rfc.abstract) + ) + self.assertNotIn("adName", upserted_dict) + self.assertNotIn("content", upserted_dict) # no blob + self.assertNotIn("subseries", upserted_dict) + + # repeat, this time with contents, an AD, and subseries docs + mock_upsert.reset_mock() + store_str( + kind="rfc", + name=f"txt/{rfc.name}.txt", + content="The contents of this RFC", + doc_name=rfc.name, + doc_rev=rfc.rev, # expected to be None + ) + rfc.ad = PersonFactory(name="Alfred D. Rector") + # Put it in two Subseries docs to be sure this does not break things + # (the typesense schema does not support this for real at the moment) + BcpFactory(contains=[rfc], name="bcp1234") + StdFactory(contains=[rfc], name="std1234") + searchindex.update_or_create_rfc_entry(rfc) + self.assertTrue(mock_upsert.called) + upserted_dict = mock_upsert.call_args[0][0] + # Check a few values, not exhaustive + self.assertEqual( + upserted_dict["content"], + searchindex._sanitize_text("The contents of this RFC"), + ) + self.assertEqual(upserted_dict["adName"], "Alfred D. Rector") + self.assertIn("subseries", upserted_dict) + ss_dict = upserted_dict["subseries"] + # We should get one of the two subseries docs, but neither is more correct + # than the other... + self.assertTrue( + any( + ss_dict == {"acronym": ss_type, "number": 1234, "total": 1} + for ss_type in ["bcp", "std"] + ) + ) + + # Finally, delete the contents blob and make sure things don't blow up + mock_upsert.reset_mock() + Blob.objects.get(bucket="rfc", name=f"txt/{rfc.name}.txt").delete() + searchindex.update_or_create_rfc_entry(rfc) + self.assertTrue(mock_upsert.called) + upserted_dict = mock_upsert.call_args[0][0] + self.assertNotIn("content", upserted_dict) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 56e395c5ac..8c0c66cdf2 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2024, All Rights Reserved +# Copyright The IETF Trust 2007-2026, All Rights Reserved # -*- coding: utf-8 -*- from base64 import b64decode @@ -443,12 +443,8 @@ def _multiline_to_list(s): ), } RFCINDEX_DELETE_THEN_WRITE = False # S3Storage allows file_overwrite by default -RFCINDEX_OUTPUT_PATH = os.environ.get( - "DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/" -) -RFCINDEX_INPUT_PATH = os.environ.get( - "DATATRACKER_RFCINDEX_INPUT_PATH", "" -) +RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/") +RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH", "") # Configure the blobdb app for artifact storage _blobdb_replication_enabled = ( @@ -471,3 +467,13 @@ def _multiline_to_list(s): PASSWORD_POLICY_ENFORCE_AT_LOGIN = ( os.environ.get("DATATRACKER_ENFORCE_PW_POLICY", "true").lower() != "false" ) + +# Typesense search indexing +SEARCHINDEX_CONFIG = { + "TYPESENSE_API_URL": os.environ.get("DATATRACKER_TYPESENSE_API_URL", ""), + "TYPESENSE_API_KEY": os.environ.get("DATATRACKER_TYPESENSE_API_KEY", ""), + "TASK_RETRY_DELAY": os.environ.get("DATATRACKER_SEARCHINDEX_TASK_RETRY_DELAY", 10), + "TASK_MAX_RETRIES": os.environ.get( + "DATATRACKER_SEARCHINDEX_TASK_MAX_RETRIES", "12" + ), +} diff --git a/requirements.txt b/requirements.txt index 3d54b104ee..2b8185dab9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -75,6 +75,7 @@ pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=2.0.0 # from TastyPie pytz==2025.2 # Pinned as changes need to be vetted for their effect on Meeting fields types-pytz==2025.2.0.20251108 # match pytz version +typesense>=2.0.0 requests>=2.32.4 types-requests>=2.32.4 requests-mock>=1.12.1 From e6a3b3ebc03ef539454cfa154ad0242b32c6a335 Mon Sep 17 00:00:00 2001 From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com> Date: Mon, 23 Mar 2026 15:46:07 +0000 Subject: [PATCH 096/158] ci: update base image target version to 20260323T1533 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index ce1828052e..af43e990e0 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260304T1633 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260323T1533 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 6be54fb6b0..09f74cce28 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260304T1633 +20260323T1533 From 33f0dbf9e969a233f46251909515b249330fbb79 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 23 Mar 2026 12:39:15 -0500 Subject: [PATCH 097/158] feat: trigger red recomputation on RFC publication or metadata update (#10567) * feat: trigger red recomputation on RFC publication or metadata update * fix: move red precomputer call out of transaction * chore: remove old comment, simplify request call * fix: isolate delayed task in test * test: give settings_test an InMemoryStorage for r2-rfc * fix: follow obs/updates both ways when notifying red of changes * fix: improve red utils, test red and r2 utils * chore: ruff * chore: remove unused import * test: fix patch paths --------- Co-authored-by: Jennifer Richards --- ietf/api/serializers_rpc.py | 15 +++- ietf/api/tests_serializers_rpc.py | 88 ++++++++++++++++--- ietf/api/tests_views_rpc.py | 24 ++++- ietf/api/views_rpc.py | 12 ++- ietf/doc/tasks.py | 17 ++++ ietf/doc/tests_utils.py | 140 +++++++++++++++++++++++++++++- ietf/doc/utils_r2.py | 17 ++++ ietf/doc/utils_red.py | 31 +++++++ ietf/settings_test.py | 6 +- k8s/settings_local.py | 10 +++ 10 files changed, 341 insertions(+), 19 deletions(-) create mode 100644 ietf/doc/utils_r2.py create mode 100644 ietf/doc/utils_red.py diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 701f05eece..397ca05d9b 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -20,7 +20,7 @@ RfcAuthor, ) from ietf.doc.serializers import RfcAuthorSerializer -from ietf.doc.tasks import update_rfc_searchindex_task +from ietf.doc.tasks import trigger_red_precomputer_task, update_rfc_searchindex_task from ietf.doc.utils import ( default_consensus, prettify_std_name, @@ -683,7 +683,18 @@ def update(self, instance, validated_data): stale_subseries_relations.delete() if len(rfc_events) > 0: rfc.save_with_history(rfc_events) - + # Gather obs and updates in both directions as a title/author change to + # this doc affects the info rendering of all of the other RFCs + needs_updating = sorted( + [ + d.rfc_number + for d in [rfc] + + rfc.related_that_doc(("obs", "updates")) + + rfc.related_that(("obs", "updates")) + ] + ) + trigger_red_precomputer_task.delay(rfc_number_list=needs_updating) + # Update the search index also update_rfc_searchindex_task.delay(rfc.rfc_number) return rfc diff --git a/ietf/api/tests_serializers_rpc.py b/ietf/api/tests_serializers_rpc.py index ed326be451..167ffcd3ee 100644 --- a/ietf/api/tests_serializers_rpc.py +++ b/ietf/api/tests_serializers_rpc.py @@ -1,4 +1,5 @@ # Copyright The IETF Trust 2026, All Rights Reserved + from unittest import mock from django.utils import timezone @@ -35,8 +36,21 @@ def test_create(self): serializer.save() @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") - def test_update(self, mock_update_searchindex_task): + @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task") + def test_update(self, mock_trigger_red_task, mock_update_searchindex_task): + updates = WgRfcFactory.create_batch(2) + obsoletes = WgRfcFactory.create_batch(2) rfc = WgRfcFactory(pages=10) + updated_by = WgRfcFactory.create_batch(2) + obsoleted_by = WgRfcFactory.create_batch(2) + for d in updates: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in obsoletes: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in updated_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + for d in obsoleted_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) serializer = EditableRfcSerializer( instance=rfc, data={ @@ -59,11 +73,6 @@ def test_update(self, mock_update_searchindex_task): ) self.assertTrue(serializer.is_valid()) result = serializer.save() - self.assertTrue(mock_update_searchindex_task.delay.called) - self.assertEqual( - mock_update_searchindex_task.delay.call_args, - mock.call(rfc.rfc_number), - ) result.refresh_from_db() self.assertEqual(result.title, "Yadda yadda yadda") self.assertEqual( @@ -91,12 +100,42 @@ def test_update(self, mock_update_searchindex_task): result.part_of(), [Document.objects.get(name="fyi999")], ) + # Confirm that red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_numbers = sorted( + [ + d.rfc_number + for d in [rfc] + updates + obsoletes + updated_by + obsoleted_by + ] + ) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was triggered correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") - def test_partial_update(self, mock_update_searchindex_task): + @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task") + def test_partial_update(self, mock_trigger_red_task, mock_update_searchindex_task): # We could test other permutations of fields, but authors is a partial update # we know we are going to use, so verifying that one in particular. + updates = WgRfcFactory.create_batch(2) + obsoletes = WgRfcFactory.create_batch(2) rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan") + updated_by = WgRfcFactory.create_batch(2) + obsoleted_by = WgRfcFactory.create_batch(2) + for d in updates: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in obsoletes: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in updated_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + for d in obsoleted_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) serializer = EditableRfcSerializer( partial=True, instance=rfc, @@ -113,11 +152,6 @@ def test_partial_update(self, mock_update_searchindex_task): ) self.assertTrue(serializer.is_valid()) result = serializer.save() - self.assertTrue(mock_update_searchindex_task.delay.called) - self.assertEqual( - mock_update_searchindex_task.delay.call_args, - mock.call(rfc.rfc_number), - ) result.refresh_from_db() self.assertEqual(rfc.title, "padawan") self.assertEqual( @@ -140,8 +174,27 @@ def test_partial_update(self, mock_update_searchindex_task): self.assertEqual(result.pages, 10) self.assertEqual(result.std_level_id, "ps") self.assertEqual(result.part_of(), []) + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_numbers = sorted( + [ + d.rfc_number + for d in [rfc] + updates + obsoletes + updated_by + obsoleted_by + ] + ) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) # Test only a field on the Document itself to be sure that it works + mock_trigger_red_task.delay.reset_mock() + mock_update_searchindex_task.delay.reset_mock() serializer = EditableRfcSerializer( partial=True, instance=rfc, @@ -151,3 +204,14 @@ def test_partial_update(self, mock_update_searchindex_task): result = serializer.save() result.refresh_from_db() self.assertEqual(rfc.title, "jedi master") + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index a679e74789..6d10bee8e8 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -197,7 +197,8 @@ def test_notify_rfc_published(self, mock_task_delay): @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task") - def test_upload_rfc_files(self, mock_update_searchindex_task): + @mock.patch("ietf.api.views_rpc.trigger_red_precomputer_task") + def test_upload_rfc_files(self, mock_trigger_red_task, mock_update_searchindex_task): def _valid_post_data(): """Generate a valid post data dict @@ -218,7 +219,14 @@ def _valid_post_data(): } url = urlreverse("ietf.api.purple_api.upload_rfc_files") + updates = RfcFactory.create_batch(2) + obsoletes = RfcFactory.create_batch(2) + rfc = WgRfcFactory() + for r in obsoletes: + rfc.relateddocument_set.create(relationship_id="obs", target=r) + for r in updates: + rfc.relateddocument_set.create(relationship_id="updates", target=r) assert isinstance(rfc, Document), "WgRfcFactory should generate a Document" with TemporaryDirectory() as rfc_dir: settings.RFC_PATH = rfc_dir # affects overridden settings @@ -303,6 +311,7 @@ def _valid_post_data(): blob_in_the_way.delete() # valid post + mock_trigger_red_task.delay.reset_mock() r = self.client.post( url, _valid_post_data(), @@ -310,7 +319,6 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 200) - self.assertTrue(mock_update_searchindex_task.delay.called) self.assertEqual( mock_update_searchindex_task.delay.call_args, mock.call(rfc.rfc_number), @@ -350,6 +358,18 @@ def _valid_post_data(): b"This is .notprepped.xml", ".notprepped.xml blob should contain the expected content", ) + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_rfc_number_list = [rfc.rfc_number] + expected_rfc_number_list.extend( + [d.rfc_number for d in updates + obsoletes] + ) + expected_rfc_number_list = sorted(set(expected_rfc_number_list)) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) # re-post with replace = False should now fail mock_update_searchindex_task.reset_mock() diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index cb6a59a167..59eed1e10e 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -38,7 +38,11 @@ from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent from ietf.doc.serializers import RfcAuthorSerializer from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage -from ietf.doc.tasks import signal_update_rfc_metadata_task, update_rfc_searchindex_task +from ietf.doc.tasks import ( + signal_update_rfc_metadata_task, + trigger_red_precomputer_task, + update_rfc_searchindex_task, +) from ietf.person.models import Email, Person from ietf.sync.tasks import create_rfc_index_task @@ -516,6 +520,12 @@ def post(self, request): destination.parent.mkdir() shutil.move(ftm, destination) + # Trigger red precomputer + needs_updating = [rfc.rfc_number] + for rel in rfc.relateddocument_set.filter(relationship_id__in=["obs","updates"]): + needs_updating.append(rel.target.rfc_number) + trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating)) + # Trigger search index update update_rfc_searchindex_task.delay(rfc.rfc_number) return Response(NotificationAckSerializer().data) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index a38cd5eb5c..19edb39014 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -7,11 +7,14 @@ import debug # pyflakes:ignore from celery import shared_task +from celery.exceptions import MaxRetriesExceededError from pathlib import Path from django.conf import settings from django.utils import timezone +from ietf.doc.utils_r2 import rfcs_are_in_r2 +from ietf.doc.utils_red import trigger_red_precomputer from ietf.utils import log, searchindex from ietf.utils.timezone import datetime_today @@ -169,6 +172,20 @@ def signal_update_rfc_metadata_task(rfc_number_list=()): signal_update_rfc_metadata(rfc_number_list) +@shared_task(bind=True) +def trigger_red_precomputer_task(self, rfc_number_list=()): + if not rfcs_are_in_r2(rfc_number_list): + log.log(f"Objects are not yet in R2 for RFCs {rfc_number_list}") + try: + countdown = getattr(settings, "RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", 10) + max_retries = getattr(settings, "RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", 12) + self.retry(countdown=countdown, max_retries=max_retries) + except MaxRetriesExceededError: + log.log(f"Gave up waiting for objects in R2 for RFCs {rfc_number_list}") + else: + trigger_red_precomputer(rfc_number_list) + + @shared_task(bind=True) def update_rfc_searchindex_task(self, rfc_number: int): """Update the search index for one RFC""" diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py index a2784bc85e..ba672cd847 100644 --- a/ietf/doc/tests_utils.py +++ b/ietf/doc/tests_utils.py @@ -1,15 +1,23 @@ # Copyright The IETF Trust 2020, All Rights Reserved import datetime +from io import BytesIO + +import mock import debug # pyflakes:ignore +import requests from pathlib import Path from unittest.mock import call, patch from django.conf import settings +from django.core.files.storage import storages from django.db import IntegrityError from django.test.utils import override_settings from django.utils import timezone + +from ietf.doc.utils_r2 import rfcs_are_in_r2 +from ietf.doc.utils_red import trigger_red_precomputer from ietf.group.factories import GroupFactory, RoleFactory from ietf.name.models import DocTagName from ietf.person.factories import PersonFactory @@ -17,11 +25,12 @@ from ietf.utils.test_utils import TestCase, name_of_file_containing, reload_db_objects from ietf.person.models import Person from ietf.doc.factories import DocumentFactory, WgRfcFactory, WgDraftFactory -from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor +from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor, StoredObject from ietf.doc.utils import (update_action_holders, add_state_change_event, update_documentauthors, fuzzy_find_documents, rebuild_reference_relations, build_file_urls, ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file, last_ballot_doc_revision) +from ietf.doc.storage_utils import store_str from ietf.utils.draft import Draft, PlaintextDraft from ietf.utils.xmldraft import XMLDraft @@ -559,3 +568,132 @@ def test_last_ballot_doc_revision(self): nobody = PersonFactory() self.assertIsNone(last_ballot_doc_revision(doc, nobody)) self.assertEqual(rev, last_ballot_doc_revision(doc, ad)) + + +class UtilsRedTests(TestCase): + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post") + def test_trigger_red_precomputer_not_configured(self, mock_post, mock_log): + with override_settings(): + try: + del settings.CUSTOM_SETTING_NAME + except AttributeError: + pass + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertEqual(mock_log.call_count, 1) + mock_args, _ = mock_log.call_args + self.assertEqual( + mock_args, + ("No URL configured for triggering red precompute multiple, skipping",), + ) + + mock_log.reset_mock() + with override_settings(TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL=None): + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertFalse(mock_post.called) + self.assertEqual(mock_log.call_count, 1) + mock_args, _ = mock_log.call_args + self.assertEqual( + mock_args, + ("No URL configured for triggering red precompute multiple, skipping",), + ) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + ) + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post", side_effect=requests.Timeout()) + def test_trigger_red_precomputer_swallows_timeout_exception( + self, mock_post, mock_log + ): + exception_raised = False + try: + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + except Exception: + exception_raised = True + self.assertFalse(exception_raised) + self.assertEqual(mock_log.call_count, 2) + # only checking the last log call + mock_args, _ = mock_log.call_args + self.assertEqual(len(mock_args), 1) + self.assertIn("POST request timed out", mock_args[0]) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + ) + @mock.patch("ietf.doc.utils_red.requests.post", side_effect=Exception()) + def test_trigger_red_precomputer_does_not_swallow_too_much(self, mock_post): + exception_raised = False + try: + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + except Exception: + exception_raised = True + self.assertTrue(exception_raised) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + DEFAULT_REQUESTS_TIMEOUT=314159265, + ) + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post") + def test_trigger_red_precomputer(self, mock_post, mock_log): + mock_post.return_value = mock.Mock(status_code=200) + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertTrue(mock_post.called) + _, mock_kwargs = mock_post.call_args + self.assertIn("url", mock_kwargs) + self.assertEqual(mock_kwargs["url"], "urlbits") + self.assertIn("json", mock_kwargs) + self.assertEqual(mock_kwargs["json"], {"rfcs": "1,2,3"}) + self.assertIn("timeout", mock_kwargs) + self.assertEqual(mock_kwargs["timeout"], 314159265) + self.assertEqual(mock_log.call_count, 1) # Not testing the first info log value + mock_log.reset_mock() + mock_post.reset_mock() + mock_post.return_value = mock.Mock( + status_code=500, + ) + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertEqual(mock_log.call_count, 2) + mock_args, _ = mock_log.call_args + self.assertEqual(len(mock_args), 1) + expected = f"POST request failed for {settings.TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL} : status_code=500" + self.assertEqual(mock_args[0], expected) + + +class UtilsR2TestCase(TestCase): + def test_rfcs_are_in_r2(self): + rfcs = WgRfcFactory.create_batch(2) + rfc_name_list = [rfc.name for rfc in rfcs] + rfc_number_list = [rfc.rfc_number for rfc in rfcs] + r2_rfc_bucket = storages["r2-rfc"] + # Right now the various doc Factories do not populate any content + self.assertEqual( + StoredObject.objects.filter( + store="rfc", doc_name__in=rfc_name_list + ).count(), + 0, + ) + self.assertTrue(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + for rfc in rfcs: + store_str( + kind="rfc", + name=f"testartifact/{rfc.name}.testartifact", + content="", + doc_name=rfc.name, + doc_rev=None, + ) + self.assertEqual( + StoredObject.objects.filter( + store="rfc", doc_name__in=rfc_name_list + ).count(), + 2, + ) + self.assertFalse(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + r2_rfc_bucket.save(f"testartifact/{rfcs[0].name}.testartifact", BytesIO(b"")) + self.assertFalse(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + r2_rfc_bucket.save(f"testartifact/{rfcs[1].name}.testartifact", BytesIO(b"")) + self.assertTrue(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + + + diff --git a/ietf/doc/utils_r2.py b/ietf/doc/utils_r2.py new file mode 100644 index 0000000000..53fb978303 --- /dev/null +++ b/ietf/doc/utils_r2.py @@ -0,0 +1,17 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.core.files.storage import storages + +from ietf.doc.models import StoredObject + + +def rfcs_are_in_r2(rfc_number_list=()): + r2_rfc_bucket = storages["r2-rfc"] + for rfc_number in rfc_number_list: + stored_objects = StoredObject.objects.filter( + store="rfc", doc_name=f"rfc{rfc_number}" + ) + for stored_object in stored_objects: + if not r2_rfc_bucket.exists(stored_object.name): + return False + return True diff --git a/ietf/doc/utils_red.py b/ietf/doc/utils_red.py new file mode 100644 index 0000000000..bcda893dca --- /dev/null +++ b/ietf/doc/utils_red.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import requests + +from django.conf import settings + +from ietf.utils.log import log + + +def trigger_red_precomputer(rfc_number_list=()): + url = getattr(settings, "TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None) + if url is not None: + payload = { + "rfcs": ",".join([str(n) for n in rfc_number_list]), + } + try: + log(f"Triggering red precompute multiple for RFCs {rfc_number_list}") + response = requests.post( + url=url, + json=payload, + timeout=settings.DEFAULT_REQUESTS_TIMEOUT, + ) + except requests.Timeout as e: + log(f"POST request timed out for {url} : {e}") + return + if response.status_code != 200: + log( + f"POST request failed for {url} : status_code={response.status_code}" + ) + else: + log("No URL configured for triggering red precompute multiple, skipping") diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 1f5a7e8ddc..e7ebc13eb2 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -115,8 +115,12 @@ def tempdir_with_cleanup(**kwargs): except NameError: pass -# Use InMemoryStorage for red bucket storage +# Use InMemoryStorage for red bucket and r2-rfc storages STORAGES["red_bucket"] = { "BACKEND": "django.core.files.storage.InMemoryStorage", "OPTIONS": {"location": "red_bucket"}, } +STORAGES["r2-rfc"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "r2-rfc"}, +} diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 8c0c66cdf2..323b7fd45a 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -80,6 +80,16 @@ def _multiline_to_list(s): else: raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") +_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", None) +if _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY is not None: + RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY +_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", None) +if _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES is not None: + RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES +_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = os.environ.get("DATATRACKER_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None) +if _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL is not None: + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL + # Set DEBUG if DATATRACKER_DEBUG env var is the word "true" DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" From e5b037ba83c2275efcd5a034c4bd1af67932d23f Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 23 Mar 2026 13:19:54 -0500 Subject: [PATCH 098/158] fix: rebuild reference relations once we have rfc contents (#10578) Co-authored-by: Jennifer Richards --- ietf/api/tests_views_rpc.py | 13 ++++++++++++- ietf/api/views_rpc.py | 4 ++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 6d10bee8e8..0db67e126f 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -196,9 +196,15 @@ def test_notify_rfc_published(self, mock_task_delay): self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.rebuild_reference_relations_task") @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task") @mock.patch("ietf.api.views_rpc.trigger_red_precomputer_task") - def test_upload_rfc_files(self, mock_trigger_red_task, mock_update_searchindex_task): + def test_upload_rfc_files( + self, + mock_trigger_red_task, + mock_update_searchindex_task, + mock_rebuild_relations, + ): def _valid_post_data(): """Generate a valid post data dict @@ -370,6 +376,11 @@ def _valid_post_data(): self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) # Confirm that the search index update task was called correctly self.assertTrue(mock_update_searchindex_task.delay.called) + # Confirm reference relations rebuild task was called correctly + self.assertTrue(mock_rebuild_relations.delay.called) + _, mock_kwargs = mock_rebuild_relations.delay.call_args + self.assertIn("doc_names", mock_kwargs) + self.assertEqual(mock_kwargs["doc_names"], [rfc.name]) # re-post with replace = False should now fail mock_update_searchindex_task.reset_mock() diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 59eed1e10e..6c7464e252 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -40,6 +40,7 @@ from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage from ietf.doc.tasks import ( signal_update_rfc_metadata_task, + rebuild_reference_relations_task, trigger_red_precomputer_task, update_rfc_searchindex_task, ) @@ -527,6 +528,9 @@ def post(self, request): trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating)) # Trigger search index update update_rfc_searchindex_task.delay(rfc.rfc_number) + # Trigger reference relation srebuild + rebuild_reference_relations_task.delay(doc_names=[rfc.name]) + return Response(NotificationAckSerializer().data) From 10ebdf9a6433b34d32352b4bb1b4e9b285773de8 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 23 Mar 2026 15:44:22 -0300 Subject: [PATCH 099/158] chore: deduplicate logging and clean up config (#10592) * fix: remove redundant loggers + cleanup * style: ruff ruff (logging config) * chore: alphabetize loggers * refactor: modern suppression of DisallowedHost log * style: minor cleanup / comments * fix: roll back accidental commit * fix: django.request at ERROR level * fix: squelch other SuspiciousOperation mail --- ietf/settings.py | 188 +++++++++++++++++++---------------------------- 1 file changed, 74 insertions(+), 114 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index e0b4f20118..40a4cb5c56 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2025, All Rights Reserved +# Copyright The IETF Trust 2007-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -13,6 +13,7 @@ import warnings from hashlib import sha384 from typing import Any, Dict, List, Tuple # pyflakes:ignore +from django.http import UnreadablePostError # DeprecationWarnings are suppressed by default, enable them warnings.simplefilter("always", DeprecationWarning) @@ -236,153 +237,112 @@ FILE_UPLOAD_PERMISSIONS = 0o644 -# ------------------------------------------------------------------------ -# Django/Python Logging Framework Modifications -# Filter out "Invalid HTTP_HOST" emails -# Based on http://www.tiwoc.de/blog/2013/03/django-prevent-email-notification-on-suspiciousoperation/ -from django.core.exceptions import SuspiciousOperation -def skip_suspicious_operations(record): - if record.exc_info: - exc_value = record.exc_info[1] - if isinstance(exc_value, SuspiciousOperation): - return False - return True +# +# Logging config +# -# Filter out UreadablePostError: -from django.http import UnreadablePostError +# Callback to filter out UnreadablePostError: def skip_unreadable_post(record): if record.exc_info: - exc_type, exc_value = record.exc_info[:2] # pylint: disable=unused-variable + exc_type, exc_value = record.exc_info[:2] # pylint: disable=unused-variable if isinstance(exc_value, UnreadablePostError): return False return True -# Copied from DEFAULT_LOGGING as of Django 1.10.5 on 22 Feb 2017, and modified -# to incorporate html logging, invalid http_host filtering, and more. -# Changes from the default has comments. - -# The Python logging flow is as follows: -# (see https://docs.python.org/2.7/howto/logging.html#logging-flow) -# -# Init: get a Logger: logger = logging.getLogger(name) -# -# Logging call, e.g. logger.error(level, msg, *args, exc_info=(...), extra={...}) -# --> Logger (discard if level too low for this logger) -# (create log record from level, msg, args, exc_info, extra) -# --> Filters (discard if any filter attach to logger rejects record) -# --> Handlers (discard if level too low for handler) -# --> Filters (discard if any filter attached to handler rejects record) -# --> Formatter (format log record and emit) -# - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - # - 'loggers': { - 'django': { - 'handlers': ['console', 'mail_admins'], - 'level': 'INFO', - }, - 'django.request': { - 'handlers': ['console'], - 'level': 'ERROR', + "version": 1, + "disable_existing_loggers": False, + "loggers": { + "celery": { + "handlers": ["console"], + "level": "INFO", }, - 'django.server': { - 'handlers': ['django.server'], - 'level': 'INFO', + "datatracker": { + "handlers": ["console"], + "level": "INFO", }, - 'django.security': { - 'handlers': ['console', ], - 'level': 'INFO', + "django": { + "handlers": ["console", "mail_admins"], + "level": "INFO", }, - 'oidc_provider': { - 'handlers': ['console', ], - 'level': 'DEBUG', + "django.request": {"level": "ERROR"}, # only log 5xx, ignore 4xx + "django.security": { + # SuspiciousOperation errors - log to console only + "handlers": ["console"], + "propagate": False, # no further handling please }, - 'datatracker': { - 'handlers': ['console'], - 'level': 'INFO', + "django.server": { + # Only used by Django's runserver development server + "handlers": ["django.server"], + "level": "INFO", }, - 'celery': { - 'handlers': ['console'], - 'level': 'INFO', + "oidc_provider": { + "handlers": ["console"], + "level": "DEBUG", }, }, - # - # No logger filters - # - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'plain', + "handlers": { + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "plain", }, - 'debug_console': { - # Active only when DEBUG=True - 'level': 'DEBUG', - 'filters': ['require_debug_true'], - 'class': 'logging.StreamHandler', - 'formatter': 'plain', + "debug_console": { + "level": "DEBUG", + "filters": ["require_debug_true"], + "class": "logging.StreamHandler", + "formatter": "plain", }, - 'django.server': { - 'level': 'INFO', - 'class': 'logging.StreamHandler', - 'formatter': 'django.server', + "django.server": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "django.server", }, - 'mail_admins': { - 'level': 'ERROR', - 'filters': [ - 'require_debug_false', - 'skip_suspicious_operations', # custom - 'skip_unreadable_posts', # custom + "mail_admins": { + "level": "ERROR", + "filters": [ + "require_debug_false", + "skip_unreadable_posts", ], - 'class': 'django.utils.log.AdminEmailHandler', - 'include_html': True, # non-default - } + "class": "django.utils.log.AdminEmailHandler", + "include_html": True, + }, }, - # # All these are used by handlers - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse', - }, - 'require_debug_true': { - '()': 'django.utils.log.RequireDebugTrue', + "filters": { + "require_debug_false": { + "()": "django.utils.log.RequireDebugFalse", }, - # custom filter, function defined above: - 'skip_suspicious_operations': { - '()': 'django.utils.log.CallbackFilter', - 'callback': skip_suspicious_operations, + "require_debug_true": { + "()": "django.utils.log.RequireDebugTrue", }, # custom filter, function defined above: - 'skip_unreadable_posts': { - '()': 'django.utils.log.CallbackFilter', - 'callback': skip_unreadable_post, + "skip_unreadable_posts": { + "()": "django.utils.log.CallbackFilter", + "callback": skip_unreadable_post, }, }, - # And finally the formatters - 'formatters': { - 'django.server': { - '()': 'django.utils.log.ServerFormatter', - 'format': '[%(server_time)s] %(message)s', + "formatters": { + "django.server": { + "()": "django.utils.log.ServerFormatter", + "format": "[%(server_time)s] %(message)s", }, - 'plain': { - 'style': '{', - 'format': '{levelname}: {name}:{lineno}: {message}', + "plain": { + "style": "{", + "format": "{levelname}: {name}:{lineno}: {message}", }, - 'json' : { + "json": { "class": "ietf.utils.jsonlogger.DatatrackerJsonFormatter", "style": "{", - "format": "{asctime}{levelname}{message}{name}{pathname}{lineno}{funcName}{process}", - } + "format": ( + "{asctime}{levelname}{message}{name}{pathname}{lineno}{funcName}" + "{process}{status_code}" + ), + }, }, } -# End logging -# ------------------------------------------------------------------------ - X_FRAME_OPTIONS = 'SAMEORIGIN' CSRF_TRUSTED_ORIGINS = [ From 14dd4cfdacb49552a2fcb9d9525e713f8ebd3c26 Mon Sep 17 00:00:00 2001 From: Tianyi Gao Date: Tue, 24 Mar 2026 02:45:24 +0800 Subject: [PATCH 100/158] feat: show parents on list of teams with grouping (#8635) (#10552) * feat: show parents on list of teams with grouping (#8635) * fix: sort teams by parent type then parent name in active teams list --- ietf/group/views.py | 13 +++++++-- ietf/templates/group/active_teams.html | 38 ++++++++++++++++---------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/ietf/group/views.py b/ietf/group/views.py index efe3eca15d..8561a5059f 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -245,10 +245,19 @@ def active_review_dirs(request): return render(request, 'group/active_review_dirs.html', {'dirs' : dirs }) def active_teams(request): - teams = Group.objects.filter(type="team", state="active").order_by("name") + parent_type_order = {"area": 1, "adm": 3, None: 4} + + def team_sort_key(group): + type_id = group.parent.type_id if group.parent else None + return (parent_type_order.get(type_id, 2), group.parent.name if group.parent else "", group.name) + + teams = sorted( + Group.objects.filter(type="team", state="active").select_related("parent"), + key=team_sort_key, + ) for group in teams: group.chairs = sorted(roles(group, "chair"), key=extract_last_name) - return render(request, 'group/active_teams.html', {'teams' : teams }) + return render(request, 'group/active_teams.html', {'teams': teams}) def active_iab(request): iabgroups = Group.objects.filter(type__in=("program","iabasg","iabworkshop"), state="active").order_by("-type_id","name") diff --git a/ietf/templates/group/active_teams.html b/ietf/templates/group/active_teams.html index 502d971a20..771dfda290 100644 --- a/ietf/templates/group/active_teams.html +++ b/ietf/templates/group/active_teams.html @@ -16,21 +16,29 @@

        Active teams

         - - {% for group in teams %} - - - - - - {% endfor %} - + {% regroup teams by parent as grouped_teams %} + {% for group_entry in grouped_teams %} + + + + + {% for group in group_entry.list %} + + + + + + {% endfor %} + + {% endfor %}
        Parent + {{ group.parent.name }} + ({{ group.parent.acronym }}) +
        Chairs
        - {{ group.acronym }} - {{ group.name }} - {% for chair in group.chairs %} - {% person_link chair.person %}{% if not forloop.last %},{% endif %} - {% endfor %} -
        + {% if group_entry.grouper %}{{ group_entry.grouper.name }}{% else %}Other{% endif %} +
        + {{ group.acronym }} + {{ group.name }} + {% for chair in group.chairs %} + {% person_link chair.person %}{% if not forloop.last %},{% endif %} + {% endfor %} +
        {% endblock %} {% block js %} From 057d52b76666ab6fcfd366700862be10db4844bb Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 23 Mar 2026 15:56:39 -0300 Subject: [PATCH 101/158] ci: update actions to avoid deprecations (#10604) * ci: upload-artifact -> v7 * ci: checkout -> v6 --- .github/workflows/build-base-app.yml | 2 +- .github/workflows/build-devblobstore.yml | 2 +- .github/workflows/build-mq-broker.yml | 2 +- .github/workflows/build.yml | 8 ++++---- .github/workflows/ci-run-tests.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/dev-assets-sync-nightly.yml | 2 +- .github/workflows/tests.yml | 14 +++++++------- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 4a4394fca0..2b937cbfef 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -18,7 +18,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: token: ${{ secrets.GH_COMMON_TOKEN }} diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml index f49a11af19..41b2e0d47a 100644 --- a/.github/workflows/build-devblobstore.yml +++ b/.github/workflows/build-devblobstore.yml @@ -20,7 +20,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index 4de861dbcd..76c9b93168 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -24,7 +24,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d97889fbb8..8872c7f7d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,7 +66,7 @@ jobs: base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 fetch-tags: false @@ -164,7 +164,7 @@ jobs: TARGET_BASE: ${{needs.prepare.outputs.base_image_version}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 fetch-tags: false @@ -341,7 +341,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Upload Build Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: release-${{ env.PKG_VERSION }} path: /home/runner/work/release/release.tar.gz @@ -403,7 +403,7 @@ jobs: PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: ref: main diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml index 278bd8af2f..5349f1ac7a 100644 --- a/.github/workflows/ci-run-tests.yml +++ b/.github/workflows/ci-run-tests.yml @@ -23,7 +23,7 @@ jobs: base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 fetch-tags: false diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3444c03b5e..4ab32d27a6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Initialize CodeQL uses: github/codeql-action/init@v3 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 6d0683c471..e255b270ff 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: 'Dependency Review' uses: actions/dependency-review-action@v4 with: diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index 4cfbf6365b..926d816b38 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -29,7 +29,7 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Login to GitHub Container Registry uses: docker/login-action@v3 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 836314bac0..be7b834b7a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,7 +32,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-devblobstore:latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Prepare for tests run: | @@ -68,7 +68,7 @@ jobs: coverage xml - name: Upload geckodriver.log - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ failure() }} with: name: geckodriverlog @@ -87,7 +87,7 @@ jobs: mv latest-coverage.json coverage.json - name: Upload Coverage Results as Build Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} with: name: coverage @@ -102,7 +102,7 @@ jobs: project: [chromium, firefox] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: actions/setup-node@v6 with: @@ -121,7 +121,7 @@ jobs: npx playwright test --project=${{ matrix.project }} - name: Upload Report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} continue-on-error: true with: @@ -143,7 +143,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-db:latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Prepare for tests run: | @@ -180,7 +180,7 @@ jobs: npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js - name: Upload Report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} continue-on-error: true with: From 02070ee2f4dc6ee599e08a87e92b345198ae40fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 16:04:08 -0300 Subject: [PATCH 102/158] chore(deps): bump actions/setup-python from 5 to 6 (#9480) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8872c7f7d3..07a304cac2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -175,7 +175,7 @@ jobs: node-version: 18.x - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.x" From 4a6627826993863bcb9cd5ede6b6e6f5b19eb0e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 17:07:28 -0300 Subject: [PATCH 103/158] chore(deps): bump ncipollo/release-action from 1.18.0 to 1.20.0 (#9478) Bumps [ncipollo/release-action](https://github.com/ncipollo/release-action) from 1.18.0 to 1.20.0. - [Release notes](https://github.com/ncipollo/release-action/releases) - [Commits](https://github.com/ncipollo/release-action/compare/v1.18.0...v1.20.0) --- updated-dependencies: - dependency-name: ncipollo/release-action dependency-version: 1.20.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 07a304cac2..ed425f9ae5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -98,7 +98,7 @@ jobs: echo "IS_RELEASE=true" >> $GITHUB_ENV - name: Create Draft Release - uses: ncipollo/release-action@v1.18.0 + uses: ncipollo/release-action@v1.21.0 if: ${{ github.ref_name == 'release' }} with: prerelease: true @@ -315,7 +315,7 @@ jobs: histCoveragePath: historical-coverage.json - name: Create Release - uses: ncipollo/release-action@v1.18.0 + uses: ncipollo/release-action@v1.21.0 if: ${{ env.SHOULD_DEPLOY == 'true' }} with: allowUpdates: true @@ -328,7 +328,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Update Baseline Coverage - uses: ncipollo/release-action@v1.18.0 + uses: ncipollo/release-action@v1.21.0 if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }} with: allowUpdates: true From 753bd507c5d9cfdad4793d0e3feed68726fecf1e Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 23 Mar 2026 15:38:15 -0500 Subject: [PATCH 104/158] fix: include editorial docs in sent-to-rpc (#10605) --- ietf/api/views_rpc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 6c7464e252..1e96118e58 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -210,19 +210,19 @@ def submitted_to_rpc(self, request): Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. """ ietf_docs = Q(states__type_id="draft-iesg", states__slug__in=["ann"]) - irtf_iab_ise_docs = Q( + irtf_iab_ise_editorial_docs = Q( states__type_id__in=[ "draft-stream-iab", "draft-stream-irtf", "draft-stream-ise", + "draft-stream-editorial", ], states__slug__in=["rfc-edit"], ) - # TODO: Need a way to talk about editorial stream docs docs = ( self.get_queryset() .filter(type_id="draft") - .filter(ietf_docs | irtf_iab_ise_docs) + .filter(ietf_docs | irtf_iab_ise_editorial_docs) ) serializer = self.get_serializer(docs, many=True) return Response(serializer.data) From 4308162174bb565b988c4fca9289c424c736ecba Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 23 Mar 2026 17:38:34 -0300 Subject: [PATCH 105/158] ci: handle rabbitmq version for push trigger (#10606) --- .github/workflows/build-mq-broker.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index 76c9b93168..50472122c4 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -39,6 +39,15 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Set rabbitmq version + id: rabbitmq-version + run: | + if [[ "${{ inputs.rabbitmq_version }}" == "" ]]; then + echo "RABBITMQ_VERSION=3.13-alpine" >> $GITHUB_OUTPUT + else + echo "RABBITMQ_VERSION=${{ inputs.rabbitmq_version }}" >> $GITHUB_OUTPUT + fi + - name: Docker Build & Push uses: docker/build-push-action@v6 env: @@ -48,7 +57,7 @@ jobs: file: dev/mq/Dockerfile platforms: linux/amd64,linux/arm64 push: true - build-args: RABBITMQ_VERSION=${{ inputs.rabbitmq_version }} + build-args: RABBITMQ_VERSION=${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }} tags: | - ghcr.io/ietf-tools/datatracker-mq:${{ inputs.rabbitmq_version }} + ghcr.io/ietf-tools/datatracker-mq:${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }} ghcr.io/ietf-tools/datatracker-mq:latest From eb041f7d81c7469f0b4c765150ccfadba5604177 Mon Sep 17 00:00:00 2001 From: Martin Thomson Date: Tue, 24 Mar 2026 05:39:01 +0900 Subject: [PATCH 106/158] fix: Rewrite CSS style attributes in SVG (#10584) This makes the dark mode work properly for drafts like https://datatracker.ietf.org/doc/html/draft-hajdusek-qirg-timing-physics-01 which have diagrams that use a mix of ordinary attributes and the style attribute. Using the style attribute makes the rules there invisible to the method we use for the remapping of black and white for dark mode. --- ietf/static/js/document_html.js | 79 +++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/ietf/static/js/document_html.js b/ietf/static/js/document_html.js index 6e8861739a..3e609f3965 100644 --- a/ietf/static/js/document_html.js +++ b/ietf/static/js/document_html.js @@ -117,4 +117,83 @@ document.addEventListener("DOMContentLoaded", function (event) { } }); } + + // Rewrite these CSS properties so that the values are available for restyling. + document.querySelectorAll("svg [style]").forEach(el => { + // Push these CSS properties into their own attributes + const SVG_PRESENTATION_ATTRS = new Set([ + 'alignment-baseline', 'baseline-shift', 'clip', 'clip-path', 'clip-rule', + 'color', 'color-interpolation', 'color-interpolation-filters', + 'color-rendering', 'cursor', 'direction', 'display', 'dominant-baseline', + 'fill', 'fill-opacity', 'fill-rule', 'filter', 'flood-color', + 'flood-opacity', 'font-family', 'font-size', 'font-size-adjust', + 'font-stretch', 'font-style', 'font-variant', 'font-weight', + 'image-rendering', 'letter-spacing', 'lighting-color', 'marker-end', + 'marker-mid', 'marker-start', 'mask', 'opacity', 'overflow', 'paint-order', + 'pointer-events', 'shape-rendering', 'stop-color', 'stop-opacity', + 'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', + 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', 'stroke-width', + 'text-anchor', 'text-decoration', 'text-rendering', 'unicode-bidi', + 'vector-effect', 'visibility', 'word-spacing', 'writing-mode', + ]); + + // Simple CSS splitter: respects quoted strings and parens so semicolons + // inside url(...) or "..." don't get treated as declaration boundaries. + function parseDeclarations(styleText) { + const decls = []; + let buf = ''; + let inStr = false; + let strChar = ''; + let escaped = false; + let depth = 0; + + for (const ch of styleText) { + if (inStr) { + if (escaped) { + escaped = false; + } else if (ch === '\\') { + escaped = true; + } else if (ch === strChar) { + inStr = false; + } + } else if (ch === '"' || ch === "'") { + inStr = true; + strChar = ch; + } else if (ch === '(') { + depth++; + } else if (ch === ')') { + depth--; + } else if (ch === ';' && depth === 0) { + const trimmed = buf.trim(); + if (trimmed) { + decls.push(trimmed); + } + buf = ''; + continue; + } + buf += ch; + } + const trimmed = buf.trim(); + if (trimmed) { + decls.push(trimmed); + } + return decls; + } + + const remainder = []; + for (const decl of parseDeclarations(el.getAttribute('style'))) { + const [prop, val] = decl.split(":", 2).map(v => v.trim()); + if (val && !/!important$/.test(val) && SVG_PRESENTATION_ATTRS.has(prop)) { + el.setAttribute(prop, val); + } else { + remainder.push(decl); + } + } + + if (remainder.length > 0) { + el.setAttribute('style', remainder.join('; ')); + } else { + el.removeAttribute('style'); + } + }); }); From 93e9bd3aad53808e791302c8fd99d74eb0873385 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 17:44:26 -0300 Subject: [PATCH 107/158] chore(deps): bump github/codeql-action from 3 to 4 (#9956) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3 to 4. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v3...v4) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4ab32d27a6..bc20779ae6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -29,9 +29,9 @@ jobs: uses: actions/checkout@v6 - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 From f9aebd5aa881557d7493db415d04a3d89494d637 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 17:48:56 -0300 Subject: [PATCH 108/158] chore(deps): bump actions/download-artifact from 4.3.0 to 6.0.0 (#9805) Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.3.0 to 6.0.0. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v4.3.0...v6.0.0) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: 6.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed425f9ae5..74791747b6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -186,7 +186,7 @@ jobs: - name: Download a Coverage Results if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v4.3.0 + uses: actions/download-artifact@v6.0.0 with: name: coverage @@ -291,7 +291,7 @@ jobs: - name: Download Coverage Results if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v4.3.0 + uses: actions/download-artifact@v6.0.0 with: name: coverage From 7d84aacad621b83753d6701afbcc864592d4822f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 17:50:26 -0300 Subject: [PATCH 109/158] chore(deps): bump the npm group (#10602) Bumps the npm group in /dev/deploy-to-container with 2 updates: [dockerode](https://github.com/apocas/dockerode) and [tar](https://github.com/isaacs/node-tar). Updates `dockerode` from 4.0.9 to 4.0.10 - [Release notes](https://github.com/apocas/dockerode/releases) - [Commits](https://github.com/apocas/dockerode/compare/v4.0.9...v4.0.10) Updates `tar` from 7.5.11 to 7.5.12 - [Release notes](https://github.com/isaacs/node-tar/releases) - [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md) - [Commits](https://github.com/isaacs/node-tar/compare/v7.5.11...v7.5.12) --- updated-dependencies: - dependency-name: dockerode dependency-version: 4.0.10 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: npm - dependency-name: tar dependency-version: 7.5.12 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: npm ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dev/deploy-to-container/package-lock.json | 113 ++++++++++------------ dev/deploy-to-container/package.json | 4 +- 2 files changed, 54 insertions(+), 63 deletions(-) diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json index b62109f0e2..a68f170c4b 100644 --- a/dev/deploy-to-container/package-lock.json +++ b/dev/deploy-to-container/package-lock.json @@ -6,12 +6,12 @@ "": { "name": "deploy-to-container", "dependencies": { - "dockerode": "^4.0.9", + "dockerode": "^4.0.10", "fs-extra": "^11.3.4", "nanoid": "5.1.7", "nanoid-dictionary": "5.0.0", "slugify": "1.6.8", - "tar": "^7.5.11", + "tar": "^7.5.12", "yargs": "^17.7.2" }, "engines": { @@ -160,7 +160,6 @@ "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", - "license": "MIT", "dependencies": { "safer-buffer": "~2.1.0" } @@ -188,7 +187,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", - "license": "BSD-3-Clause", "dependencies": { "tweetnacl": "^0.14.3" } @@ -227,9 +225,9 @@ } }, "node_modules/buildcheck": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", - "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", "optional": true, "engines": { "node": ">=10.0.0" @@ -284,10 +282,9 @@ } }, "node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "license": "MIT", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "dependencies": { "ms": "^2.1.3" }, @@ -301,10 +298,9 @@ } }, "node_modules/docker-modem": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz", - "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", - "license": "Apache-2.0", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", "dependencies": { "debug": "^4.1.1", "readable-stream": "^3.5.0", @@ -316,14 +312,14 @@ } }, "node_modules/dockerode": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz", - "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==", + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.10.tgz", + "integrity": "sha512-8L/P9JynLBiG7/coiA4FlQXegHltRqS0a+KqI44P1zgQh8QLHTg7FKOwhkBgSJwZTeHsq30WRoVFLuwkfK0YFg==", "dependencies": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", - "docker-modem": "^5.0.6", + "docker-modem": "^5.0.7", "protobufjs": "^7.3.2", "tar-fs": "^2.1.4", "uuid": "^10.0.0" @@ -464,14 +460,12 @@ "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/nan": { - "version": "2.22.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", - "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", - "license": "MIT", + "version": "2.26.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz", + "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==", "optional": true }, "node_modules/nanoid": { @@ -580,8 +574,7 @@ "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/slugify": { "version": "1.6.8", @@ -594,13 +587,12 @@ "node_modules/split-ca": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", - "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", - "license": "ISC" + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" }, "node_modules/ssh2": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", - "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", "hasInstallScript": true, "dependencies": { "asn1": "^0.2.6", @@ -611,7 +603,7 @@ }, "optionalDependencies": { "cpu-features": "~0.0.10", - "nan": "^2.20.0" + "nan": "^2.23.0" } }, "node_modules/string_decoder": { @@ -647,9 +639,9 @@ } }, "node_modules/tar": { - "version": "7.5.11", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz", - "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==", + "version": "7.5.12", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.12.tgz", + "integrity": "sha512-9TsuLcdhOn4XztcQqhNyq1KOwOOED/3k58JAvtULiYqbO8B/0IBAAIE1hj0Svmm58k27TmcigyDI0deMlgG3uw==", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", @@ -698,8 +690,7 @@ "node_modules/tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", - "license": "Unlicense" + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, "node_modules/undici-types": { "version": "6.20.0", @@ -949,9 +940,9 @@ } }, "buildcheck": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", - "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", "optional": true }, "chownr": { @@ -993,17 +984,17 @@ } }, "debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "requires": { "ms": "^2.1.3" } }, "docker-modem": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz", - "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", "requires": { "debug": "^4.1.1", "readable-stream": "^3.5.0", @@ -1012,14 +1003,14 @@ } }, "dockerode": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz", - "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==", + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.10.tgz", + "integrity": "sha512-8L/P9JynLBiG7/coiA4FlQXegHltRqS0a+KqI44P1zgQh8QLHTg7FKOwhkBgSJwZTeHsq30WRoVFLuwkfK0YFg==", "requires": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", - "docker-modem": "^5.0.6", + "docker-modem": "^5.0.7", "protobufjs": "^7.3.2", "tar-fs": "^2.1.4", "uuid": "^10.0.0" @@ -1126,9 +1117,9 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "nan": { - "version": "2.22.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", - "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", + "version": "2.26.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz", + "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==", "optional": true }, "nanoid": { @@ -1213,14 +1204,14 @@ "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" }, "ssh2": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", - "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", "requires": { "asn1": "^0.2.6", "bcrypt-pbkdf": "^1.0.2", "cpu-features": "~0.0.10", - "nan": "^2.20.0" + "nan": "^2.23.0" } }, "string_decoder": { @@ -1250,9 +1241,9 @@ } }, "tar": { - "version": "7.5.11", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz", - "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==", + "version": "7.5.12", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.12.tgz", + "integrity": "sha512-9TsuLcdhOn4XztcQqhNyq1KOwOOED/3k58JAvtULiYqbO8B/0IBAAIE1hj0Svmm58k27TmcigyDI0deMlgG3uw==", "requires": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json index 1c95a4540c..aa9e82dbdf 100644 --- a/dev/deploy-to-container/package.json +++ b/dev/deploy-to-container/package.json @@ -2,12 +2,12 @@ "name": "deploy-to-container", "type": "module", "dependencies": { - "dockerode": "^4.0.9", + "dockerode": "^4.0.10", "fs-extra": "^11.3.4", "nanoid": "5.1.7", "nanoid-dictionary": "5.0.0", "slugify": "1.6.8", - "tar": "^7.5.11", + "tar": "^7.5.12", "yargs": "^17.7.2" }, "engines": { From 3d00e594e6a667fe8084bc8548d6993c29d515e6 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 23 Mar 2026 19:50:19 -0300 Subject: [PATCH 110/158] chore(deps): bump more action versions (#10608) --- .github/workflows/build-base-app.yml | 6 +++--- .github/workflows/build-devblobstore.yml | 6 +++--- .github/workflows/build-mq-broker.yml | 6 +++--- .github/workflows/build.yml | 10 +++++----- .github/workflows/dev-assets-sync-nightly.yml | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 2b937cbfef..1b0855cc47 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -31,17 +31,17 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml index 41b2e0d47a..14c4b1a135 100644 --- a/.github/workflows/build-devblobstore.yml +++ b/.github/workflows/build-devblobstore.yml @@ -23,17 +23,17 @@ jobs: - uses: actions/checkout@v6 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index 50472122c4..ef7ed2f65c 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -30,10 +30,10 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} @@ -49,7 +49,7 @@ jobs: fi - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 74791747b6..8ec806b229 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -253,10 +253,10 @@ jobs: EOL - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} @@ -267,7 +267,7 @@ jobs: run: echo "FEATURE_LATEST_TAG=$(echo $GITHUB_REF_NAME | tr / -)" >> $GITHUB_ENV - name: Build Images - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: @@ -360,7 +360,7 @@ jobs: steps: - name: Notify on Slack (Success) if: ${{ !contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v2 + uses: slackapi/slack-github-action@v3 with: token: ${{ secrets.SLACK_GH_BOT }} method: chat.postMessage @@ -375,7 +375,7 @@ jobs: value: "Completed" - name: Notify on Slack (Failure) if: ${{ contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v2 + uses: slackapi/slack-github-action@v3 with: token: ${{ secrets.SLACK_GH_BOT }} method: chat.postMessage diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index 926d816b38..cd986f06f3 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -32,14 +32,14 @@ jobs: - uses: actions/checkout@v6 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: From e51469a5d437491071610156d56dcb73191ad61c Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Fri, 27 Mar 2026 13:44:35 -0400 Subject: [PATCH 111/158] feat: add email/name for ADs and WG Chairs --- ietf/api/serializers_rpc.py | 28 +++++++++++++++++++++++++++- ietf/group/serializers.py | 6 ++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 397ca05d9b..d888de4586 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -27,7 +27,7 @@ update_action_holders, update_rfcauthors, ) -from ietf.group.models import Group +from ietf.group.models import Group, Role from ietf.group.serializers import AreaSerializer from ietf.name.models import StreamName, StdLevelName from ietf.person.models import Person @@ -97,6 +97,21 @@ class Meta: fields = ["draft_name", "authors"] +class WgChairSerializer(serializers.Serializer): + """Serialize a WG chair's name and email from a Role""" + + name = serializers.SerializerMethodField() + email = serializers.SerializerMethodField() + + @extend_schema_field(serializers.CharField) + def get_name(self, role: Role) -> str: + return role.person.plain_name() + + @extend_schema_field(serializers.EmailField) + def get_email(self, role: Role) -> str: + return role.email.email_address() + + class DocumentAuthorSerializer(serializers.ModelSerializer): """Serializer for a Person in a response""" @@ -126,6 +141,7 @@ class FullDraftSerializer(serializers.ModelSerializer): source="shepherd.person", read_only=True ) consensus = serializers.SerializerMethodField() + wg_chairs = serializers.SerializerMethodField() class Meta: model = Document @@ -145,11 +161,21 @@ class Meta: "consensus", "shepherd", "ad", + "wg_chairs", ] def get_consensus(self, doc: Document) -> Optional[bool]: return default_consensus(doc) + @extend_schema_field(WgChairSerializer(many=True)) + def get_wg_chairs(self, doc: Document): + if doc.group is None: + return [] + chairs = doc.group.role_set.filter(name_id="chair").select_related( + "person", "email" + ) + return WgChairSerializer(chairs, many=True).data + def get_source_format( self, doc: Document ) -> Literal["unknown", "xml-v2", "xml-v3", "txt"]: diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py index db3b37af48..e789ba46bf 100644 --- a/ietf/group/serializers.py +++ b/ietf/group/serializers.py @@ -20,8 +20,14 @@ class AreaDirectorSerializer(serializers.Serializer): Works with Email or Role """ + name = serializers.SerializerMethodField() email = serializers.SerializerMethodField() + @extend_schema_field(serializers.CharField) + def get_name(self, instance: Email | Role): + person = getattr(instance, 'person', None) + return person.plain_name() if person else None + @extend_schema_field(serializers.EmailField) def get_email(self, instance: Email | Role): if isinstance(instance, Role): From b1cc7edc7ff5e80f7eb0072657e88450a4b2c06b Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Fri, 27 Mar 2026 14:32:07 -0400 Subject: [PATCH 112/158] adapt test --- ietf/group/tests_serializers.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ietf/group/tests_serializers.py b/ietf/group/tests_serializers.py index bf29e6c8fd..b584a17ae2 100644 --- a/ietf/group/tests_serializers.py +++ b/ietf/group/tests_serializers.py @@ -31,7 +31,7 @@ def test_serializes_role(self): serialized = AreaDirectorSerializer(role).data self.assertEqual( serialized, - {"email": role.email.email_address()}, + {"email": role.email.email_address(), "name": role.person.plain_name()}, ) def test_serializes_email(self): @@ -40,7 +40,10 @@ def test_serializes_email(self): serialized = AreaDirectorSerializer(email).data self.assertEqual( serialized, - {"email": email.email_address()}, + { + "email": email.email_address(), + "name": email.person.plain_name() if email.person else None, + }, ) @@ -63,7 +66,10 @@ def test_serializes_active_area(self): self.assertEqual(serialized["name"], area.name) self.assertCountEqual( serialized["ads"], - [{"email": ad.email.email_address()} for ad in ad_roles], + [ + {"email": ad.email.email_address(), "name": ad.person.plain_name()} + for ad in ad_roles + ], ) def test_serializes_inactive_area(self): From 5775077317640de8981cf27b0b8c54e42d8ae9a2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 1 Apr 2026 18:53:13 -0300 Subject: [PATCH 113/158] fix: limit access to manual post cancellation (#10638) * fix: drop access_token from URL * test: update test case * test: remove unneeded test There is no longer a dedicated manual post cancel action * chore: update copyrights --- ietf/submit/tests.py | 30 +++++++++++++++----------- ietf/templates/submit/manual_post.html | 16 ++++---------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 400d0d8c7d..ad361d31b2 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2011-2023, All Rights Reserved +# Copyright The IETF Trust 2011-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -207,20 +207,24 @@ def test_manualpost_view(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertIn( - urlreverse( - "ietf.submit.views.submission_status", - kwargs=dict(submission_id=submission.pk) - ), - q("#manual.submissions td a").attr("href") - ) - self.assertIn( - submission.name, - q("#manual.submissions td a").text() + # Validate that the basic submission status URL is on the manual post page + # _without_ an access token, even if logged in as various users. + expected_url = urlreverse( + "ietf.submit.views.submission_status", + kwargs=dict(submission_id=submission.pk) ) + selected_elts = q("#manual.submissions td a") + self.assertEqual(expected_url, selected_elts.attr("href")) + self.assertIn(submission.name, selected_elts.text()) + for username in ["plain", "secretary"]: + self.client.login(username=username, password=username + "+password") + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + selected_elts = q("#manual.submissions td a") + self.assertEqual(expected_url, selected_elts.attr("href")) + self.assertIn(submission.name, selected_elts.text()) - def test_manualpost_cancel(self): - pass class SubmitTests(BaseSubmitTestCase): def setUp(self): diff --git a/ietf/templates/submit/manual_post.html b/ietf/templates/submit/manual_post.html index 6e4a2ba42a..0da83e750f 100644 --- a/ietf/templates/submit/manual_post.html +++ b/ietf/templates/submit/manual_post.html @@ -1,5 +1,5 @@ {% extends "submit/submit_base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin static %} {% block pagehead %} @@ -27,17 +27,9 @@

        Submissions needing manual posting

        {% for s in manual %} - {% if user.is_authenticated %} - - - {{ s.name }}-{{ s.rev }} - - - {% else %} - - {{ s.name }}-{{ s.rev }} - - {% endif %} + + {{ s.name }}-{{ s.rev }} + {{ s.submission_date }} {% if s.passes_checks %} From 6058769a64778679d4b3b5ca5e6937ed5f2ec6c8 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 2 Apr 2026 15:57:49 -0300 Subject: [PATCH 114/158] ci: optional bucket suffix for storage cfg (#10637) * ci: optional bucket suffix for storage cfg * style: ruff ruff * fix: roll back bizarre editor glitch --- docker/scripts/app-configure-blobstore.py | 10 +++++++--- k8s/settings_local.py | 22 ++++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py index 3140e39306..9ae64e0041 100755 --- a/docker/scripts/app-configure-blobstore.py +++ b/docker/scripts/app-configure-blobstore.py @@ -24,10 +24,13 @@ def init_blobstore(): ), ) for bucketname in ARTIFACT_STORAGE_NAMES: + adjusted_bucket_name = ( + os.environ.get("BLOB_STORE_BUCKET_PREFIX", "") + + bucketname + + os.environ.get("BLOB_STORE_BUCKET_SUFFIX", "") + ).strip() try: - blobstore.create_bucket( - Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip() - ) + blobstore.create_bucket(Bucket=adjusted_bucket_name) except botocore.exceptions.ClientError as err: if err.response["Error"]["Code"] == "BucketAlreadyExists": print(f"Bucket {bucketname} already exists") @@ -36,5 +39,6 @@ def init_blobstore(): else: print(f"Bucket {bucketname} created") + if __name__ == "__main__": sys.exit(init_blobstore()) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 323b7fd45a..b45cbbe260 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -18,7 +18,7 @@ def _multiline_to_list(s): - """Helper to split at newlines and conver to list""" + """Helper to split at newlines and convert to list""" return [item.strip() for item in s.split("\n")] @@ -80,13 +80,19 @@ def _multiline_to_list(s): else: raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") -_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", None) +_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = os.environ.get( + "DATATRACKER_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", None +) if _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY is not None: - RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY -_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", None) + RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY +_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = os.environ.get( + "DATATRACKER_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", None +) if _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES is not None: RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES -_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = os.environ.get("DATATRACKER_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None) +_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = os.environ.get( + "DATATRACKER_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None +) if _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL is not None: TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL @@ -387,6 +393,7 @@ def _multiline_to_list(s): "and DATATRACKER_BLOB_STORE_SECRET_KEY must be set" ) _blob_store_bucket_prefix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "") +_blob_store_bucket_suffix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_SUFFIX", "") _blob_store_enable_profiling = ( os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true" ) @@ -406,6 +413,9 @@ def _multiline_to_list(s): if storagename in ["staging"]: continue replica_storagename = f"r2-{storagename}" + adjusted_bucket_name = ( + _blob_store_bucket_prefix + storagename + _blob_store_bucket_suffix + ).strip() STORAGES[replica_storagename] = { "BACKEND": "ietf.doc.storage.MetadataS3Storage", "OPTIONS": dict( @@ -422,7 +432,7 @@ def _multiline_to_list(s): retries={"total_max_attempts": _blob_store_max_attempts}, ), verify=False, - bucket_name=f"{_blob_store_bucket_prefix}{storagename}".strip(), + bucket_name=adjusted_bucket_name, ietf_log_blob_timing=_blob_store_enable_profiling, ), } From a46a2efc05b2e7f5d1b50c76d543e1ca16ae8918 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Tue, 7 Apr 2026 09:25:24 +1200 Subject: [PATCH 115/158] feat: Generate bcp-index.txt (#10631) * feat: Generate bcp-index.txt * fix: Fix issue with author names * feat: Update bcp-index.txt header * refactor: Generalize some functions * fix: Sort RFCs * test: Add tests for bcp-index.txt * fix: Fix range bug * test: Add test for BCP entry * test: Fix test_create_bcp_txt_index --- ietf/sync/rfcindex.py | 98 +++++++++++++++++++++++++++++++ ietf/sync/tests_rfcindex.py | 69 ++++++++++++++++++++-- ietf/templates/sync/bcp-index.txt | 52 ++++++++++++++++ 3 files changed, 215 insertions(+), 4 deletions(-) create mode 100644 ietf/templates/sync/bcp-index.txt diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index 63c2044931..357cc4069a 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -24,6 +24,8 @@ from ietf.utils.log import log FORMATS_FOR_INDEX = ["txt", "html", "pdf", "xml", "ps"] +SS_TXT_MARGIN = 3 +SS_TXT_CUE_COL_WIDTH = 14 def format_rfc_number(n): @@ -267,6 +269,87 @@ def get_rfc_text_index_entries(): return entries +def subseries_text_line(line, first=False): + """Return subseries text entry line""" + indent = " " * SS_TXT_CUE_COL_WIDTH + if first: + initial_indent = " " * SS_TXT_MARGIN + else: + initial_indent = indent + return fill( + line, + initial_indent=initial_indent, + subsequent_indent=indent, + width=80, + break_on_hyphens=False, + ) + + +def get_bcp_text_index_entries(): + """Returns BCP entries for bcp-index.txt""" + entries = [] + + highest_bcp_number = ( + Document.objects.filter(type_id="bcp") + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + .first() + .number + ) + + for bcp_number in range(1, highest_bcp_number + 1): + bcp_name = f"BCP{bcp_number}" + bcp = Document.objects.filter(type_id="bcp", name=f"{bcp_name.lower()}").first() + + if bcp: + entry = subseries_text_line( + ( + f"[{bcp_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(bcp_name) - 2 - SS_TXT_MARGIN)}" + f"Best Current Practice {bcp_number}," + ), + first=True, + ) + entry += "\n" + entry += subseries_text_line( + f"<{settings.RFC_EDITOR_INFO_BASE_URL}{bcp_name.lower()}>." + ) + entry += "\n" + entry += subseries_text_line( + "At the time of writing, this BCP comprises the following:" + ) + entry += "\n\n" + rfcs = sorted(bcp.contains(), key=lambda x: x.rfc_number) + for rfc in rfcs: + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + entry += subseries_text_line( + ( + f'{authors}, "{rfc.title}", BCP¶{bcp_number}, RFC¶{rfc.rfc_number}, ' + f"DOI¶{rfc.doi}, {rfc.pub_date().strftime('%B %Y')}, " + f"<{settings.RFC_EDITOR_INFO_BASE_URL}rfc{rfc.rfc_number}>." + ) + ).replace("¶", " ") + entry += "\n\n" + else: + entry = subseries_text_line( + ( + f"[{bcp_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(bcp_name) - 2 - SS_TXT_MARGIN)}" + f"Best Current Practice {bcp_number} currently contains no RFCs" + ), + first=True, + ) + entries.append(entry) + return entries + + def add_subseries_xml_index_entries(rfc_index, ss_type, include_all=False): """Add subseries entries for rfc-index.xml""" # subseries docs annotated with numeric number @@ -481,3 +564,18 @@ def create_rfc_xml_index(): pretty_print=4, ) save_to_red_bucket("rfc-index.xml", pretty_index) + + +def create_bcp_txt_index(): + """Create text index of BCPs""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating bcp-index.txt") + index = render_to_string( + "sync/bcp-index.txt", + { + "created_on": created_on, + "bcps": get_bcp_text_index_entries(), + }, + ) + save_to_red_bucket("bcp-index.txt", index) diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index e682c016f5..cad5b577d4 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -7,16 +7,22 @@ from django.test.utils import override_settings from lxml import etree -from ietf.doc.factories import PublishedRfcDocEventFactory, IndividualRfcFactory +from ietf.doc.factories import ( + BcpFactory, + IndividualRfcFactory, + PublishedRfcDocEventFactory, +) from ietf.name.models import DocTagName from ietf.sync.rfcindex import ( + create_bcp_txt_index, create_rfc_txt_index, create_rfc_xml_index, format_rfc_number, - save_to_red_bucket, - get_unusable_rfc_numbers, get_april1_rfc_numbers, get_publication_std_levels, + get_unusable_rfc_numbers, + save_to_red_bucket, + subseries_text_line, ) from ietf.utils.test_utils import TestCase @@ -69,6 +75,9 @@ def setUp(self): ).doc self.rfc.tags.add(DocTagName.objects.get(slug="errata")) + # Create a BCP with non-April Fools RFC + self.bcp = BcpFactory(contains=[self.rfc], name="bcp11") + # Set up a publication-std-levels.json file to indicate the publication # standard of self.rfc as different from its current value red_bucket.save( @@ -137,7 +146,7 @@ def test_create_rfc_xml_index(self, mock_save): children = list(index) # elements as list # Should be one rfc-not-issued-entry - self.assertEqual(len(children), 3) + self.assertEqual(len(children), 14) self.assertEqual( [ c.find(f"{ns}doc-id").text @@ -184,6 +193,53 @@ def test_create_rfc_xml_index(self, mock_save): [(f"{ns}month", "April"), (f"{ns}year", "2021")], ) + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_bcp_txt_index(self, mock_save): + create_bcp_txt_index() + self.assertEqual(mock_save.call_count, 1) + self.assertEqual(mock_save.call_args[0][0], "bcp-index.txt") + contents = mock_save.call_args[0][1] + self.assertTrue(isinstance(contents, str)) + # starts from 1 + self.assertIn( + "[BCP1]", + contents, + ) + # fill up to 11 + self.assertIn( + "[BCP10]", + contents, + ) + # but not to 12 + self.assertNotIn( + "[BCP12]", + contents, + ) + # Test empty BCPs + self.assertIn( + "Best Current Practice 9 currently contains no RFCs", + contents, + ) + # No zero prefix! + self.assertNotIn( + "[BCP0001]", + contents, + ) + # Has BCP11 with a RFC + self.assertIn( + "Best Current Practice 11,", + contents, + ) + self.assertIn( + f'"{self.rfc.title}"', + contents, + ) + self.assertIn( + f'BCP 11, RFC {self.rfc.rfc_number},', + contents, + ) + class HelperTests(TestCase): def test_format_rfc_number(self): @@ -234,3 +290,8 @@ def test_get_publication_std_levels_raises(self): with self.assertRaises(json.JSONDecodeError): get_publication_std_levels() red_bucket.delete("publication-std-levels.json") + + def test_subseries_text_line(self): + text = "foobar" + self.assertEqual(subseries_text_line(line=text, first=True), f" {text}") + self.assertEqual(subseries_text_line(line=text), f" {text}") diff --git a/ietf/templates/sync/bcp-index.txt b/ietf/templates/sync/bcp-index.txt new file mode 100644 index 0000000000..dd19920eba --- /dev/null +++ b/ietf/templates/sync/bcp-index.txt @@ -0,0 +1,52 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + BCP INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all BCPs in numeric order. The BCPs +form a sub-series of the RFC document series, specifically those RFCs +with the status BEST CURRENT PRACTICE. + +BCP citations appear in this format: + + [BCP#] Best Current Practice #, + . + At the time of writing, this BCP comprises the following: + + Author 1, Author 2, "Title of the RFC", BCP #, RFC №, + DOI DOI string, Issue date, + . + +For example: + + [BCP3] Best Current Practice 3, + . + At the time of writing, this BCP comprises the following: + + F. Kastenholz, "Variance for The PPP Compression Control Protocol + and The PPP Encryption Control Protocol", BCP 3, RFC 1915, + DOI 10.17487/RFC1915, February 1996, + . + +Key to fields: + +# is the BCP number. + +№ is the RFC number. + +BCPs and other RFCs may be obtained from https://www.rfc-editor.org. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + BCP INDEX + --------- + + + +{% for bcp in bcps %}{{bcp|safe}} + +{% endfor %} From 7c7219f0dcf326f369c7b4bd04337f95f0a7a9f4 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 8 Apr 2026 11:16:57 +1200 Subject: [PATCH 116/158] feat: Generate std-index.txt (#10665) * feat: Generate std-index.txt * style: Ruff ruff Good boy! * test: Fix flaky test * test: Add tests for std-index.txt --- ietf/sync/rfcindex.py | 80 +++++++++++++++++++++++++++++++ ietf/sync/tests_rfcindex.py | 64 ++++++++++++++++++++++++- ietf/templates/sync/std-index.txt | 51 ++++++++++++++++++++ 3 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 ietf/templates/sync/std-index.txt diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index 357cc4069a..6a6a4bfa9f 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -350,6 +350,71 @@ def get_bcp_text_index_entries(): return entries +def get_std_text_index_entries(): + """Returns STD entries for std-index.txt""" + entries = [] + + highest_std_number = ( + Document.objects.filter(type_id="std") + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + .first() + .number + ) + + for std_number in range(1, highest_std_number + 1): + std_name = f"STD{std_number}" + std = Document.objects.filter(type_id="std", name=f"{std_name.lower()}").first() + + if std and std.contains(): + entry = subseries_text_line( + ( + f"[{std_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(std_name) - 2 - SS_TXT_MARGIN)}" + f"Internet Standard {std_number}," + ), + first=True, + ) + entry += "\n" + entry += subseries_text_line( + f"<{settings.RFC_EDITOR_INFO_BASE_URL}{std_name.lower()}>." + ) + entry += "\n" + entry += subseries_text_line( + "At the time of writing, this STD comprises the following:" + ) + entry += "\n\n" + rfcs = sorted(std.contains(), key=lambda x: x.rfc_number) + for rfc in rfcs: + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + entry += subseries_text_line( + ( + f'{authors}, "{rfc.title}", STD¶{std_number}, RFC¶{rfc.rfc_number}, ' + f"DOI¶{rfc.doi}, {rfc.pub_date().strftime('%B %Y')}, " + f"<{settings.RFC_EDITOR_INFO_BASE_URL}rfc{rfc.rfc_number}>." + ) + ).replace("¶", " ") + entry += "\n\n" + else: + entry = subseries_text_line( + ( + f"[{std_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(std_name) - 2 - SS_TXT_MARGIN)}" + f"Internet Standard {std_number} currently contains no RFCs" + ), + first=True, + ) + entries.append(entry) + return entries + + def add_subseries_xml_index_entries(rfc_index, ss_type, include_all=False): """Add subseries entries for rfc-index.xml""" # subseries docs annotated with numeric number @@ -579,3 +644,18 @@ def create_bcp_txt_index(): }, ) save_to_red_bucket("bcp-index.txt", index) + + +def create_std_txt_index(): + """Create text index of STDs""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating std-index.txt") + index = render_to_string( + "sync/std-index.txt", + { + "created_on": created_on, + "stds": get_std_text_index_entries(), + }, + ) + save_to_red_bucket("std-index.txt", index) diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index cad5b577d4..70bc41b992 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -9,6 +9,7 @@ from ietf.doc.factories import ( BcpFactory, + StdFactory, IndividualRfcFactory, PublishedRfcDocEventFactory, ) @@ -17,6 +18,7 @@ create_bcp_txt_index, create_rfc_txt_index, create_rfc_xml_index, + create_std_txt_index, format_rfc_number, get_april1_rfc_numbers, get_publication_std_levels, @@ -78,6 +80,9 @@ def setUp(self): # Create a BCP with non-April Fools RFC self.bcp = BcpFactory(contains=[self.rfc], name="bcp11") + # Create a STD with non-April Fools RFC + self.std = StdFactory(contains=[self.rfc], name="std11") + # Set up a publication-std-levels.json file to indicate the publication # standard of self.rfc as different from its current value red_bucket.save( @@ -146,7 +151,7 @@ def test_create_rfc_xml_index(self, mock_save): children = list(index) # elements as list # Should be one rfc-not-issued-entry - self.assertEqual(len(children), 14) + self.assertEqual(len(children), 15) self.assertEqual( [ c.find(f"{ns}doc-id").text @@ -236,7 +241,62 @@ def test_create_bcp_txt_index(self, mock_save): contents, ) self.assertIn( - f'BCP 11, RFC {self.rfc.rfc_number},', + "BCP 11,", + contents, + ) + self.assertIn( + f"RFC {self.rfc.rfc_number},", + contents, + ) + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_std_txt_index(self, mock_save): + create_std_txt_index() + self.assertEqual(mock_save.call_count, 1) + self.assertEqual(mock_save.call_args[0][0], "std-index.txt") + contents = mock_save.call_args[0][1] + self.assertTrue(isinstance(contents, str)) + # starts from 1 + self.assertIn( + "[STD1]", + contents, + ) + # fill up to 11 + self.assertIn( + "[STD10]", + contents, + ) + # but not to 12 + self.assertNotIn( + "[STD12]", + contents, + ) + # Test empty STDs + self.assertIn( + "Internet Standard 9 currently contains no RFCs", + contents, + ) + # No zero prefix! + self.assertNotIn( + "[STD0001]", + contents, + ) + # Has STD11 with a RFC + self.assertIn( + "Internet Standard 11,", + contents, + ) + self.assertIn( + f'"{self.rfc.title}"', + contents, + ) + self.assertIn( + "STD 11,", + contents, + ) + self.assertIn( + f"RFC {self.rfc.rfc_number},", contents, ) diff --git a/ietf/templates/sync/std-index.txt b/ietf/templates/sync/std-index.txt new file mode 100644 index 0000000000..c075d1d43e --- /dev/null +++ b/ietf/templates/sync/std-index.txt @@ -0,0 +1,51 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + STD INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all STDs in numeric order. Each +STD represents a single Internet Standard technical specification, +composed of one or more RFCs with Internet Standard status. + +STD citations appear in this format: + + [STD#] Best Current Practice #, + . + At the time of writing, this STD comprises the following: + + Author 1, Author 2, "Title of the RFC", STD #, RFC №, + DOI DOI string, Issue date, + . + +For example: + + [STD6] Internet Standard 6, + . + At the time of writing, this STD comprises the following: + + J. Postel, "User Datagram Protocol", STD 6, RFC 768, + DOI 10.17487/RFC0768, August 1980, + . + +Key to fields: + +# is the STD number. + +№ is the RFC number. + +STDs and other RFCs may be obtained from https://www.rfc-editor.org. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + STD INDEX + --------- + + + +{% for std in stds %}{{std|safe}} + +{% endfor %} From e72ead86dee707b5cbd9aeea96437dbaee78c88d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 12:35:06 -0300 Subject: [PATCH 117/158] chore(deps): bump appleboy/ssh-action from 1.2.2 to 1.2.5 (#10623) Bumps [appleboy/ssh-action](https://github.com/appleboy/ssh-action) from 1.2.2 to 1.2.5. - [Release notes](https://github.com/appleboy/ssh-action/releases) - [Commits](https://github.com/appleboy/ssh-action/compare/2ead5e36573f08b82fbfce1504f1a4b05a647c6f...0ff4204d59e8e51228ff73bce53f80d53301dee2) --- updated-dependencies: - dependency-name: appleboy/ssh-action dependency-version: 1.2.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/tests-az.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml index 8553563a19..833ca89bef 100644 --- a/.github/workflows/tests-az.yml +++ b/.github/workflows/tests-az.yml @@ -38,7 +38,7 @@ jobs: ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts - name: Remote SSH into VM - uses: appleboy/ssh-action@2ead5e36573f08b82fbfce1504f1a4b05a647c6f + uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: From c7657c3f22f5f7a906fd2cf01aaed7b54feca9e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 12:37:35 -0300 Subject: [PATCH 118/158] chore(deps): bump stefanzweifel/git-auto-commit-action from 6 to 7 (#10624) Bumps [stefanzweifel/git-auto-commit-action](https://github.com/stefanzweifel/git-auto-commit-action) from 6 to 7. - [Release notes](https://github.com/stefanzweifel/git-auto-commit-action/releases) - [Changelog](https://github.com/stefanzweifel/git-auto-commit-action/blob/master/CHANGELOG.md) - [Commits](https://github.com/stefanzweifel/git-auto-commit-action/compare/v6...v7) --- updated-dependencies: - dependency-name: stefanzweifel/git-auto-commit-action dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-base-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 1b0855cc47..5e274838a1 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -60,7 +60,7 @@ jobs: echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE - name: Commit CHANGELOG.md - uses: stefanzweifel/git-auto-commit-action@v6 + uses: stefanzweifel/git-auto-commit-action@v7 with: branch: ${{ github.ref_name }} commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}' From f39e916a73eaab6c0172a09e98c28ba628b7bcc4 Mon Sep 17 00:00:00 2001 From: Eric Rescorla Date: Wed, 8 Apr 2026 08:50:19 -0700 Subject: [PATCH 119/158] fix: Rewrite upper right document search box (#10538) * Rewrite upper right document search box. Fixes #10358 This is a fix to the problem where the first item in the dropdown is auto-selected and then when you hit return you go to that rather than searching for what's in the text field. It appears to be challenging to get this behavior with select2, so this is actually a rewrite of the box with explicit behavior. As a side effect, the draft names actually render a bit better. Co-Authored-By: Claude Opus 4.6 * Respond to review comments --------- Co-authored-by: EKR aibot Co-authored-by: Claude Opus 4.6 --- ietf/static/css/ietf.scss | 17 +++++ ietf/static/js/navbar-doc-search.js | 113 ++++++++++++++++++++++++++++ ietf/templates/base.html | 24 +++--- package.json | 1 + 4 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 ietf/static/js/navbar-doc-search.js diff --git a/ietf/static/css/ietf.scss b/ietf/static/css/ietf.scss index df973863d5..6695c57b13 100644 --- a/ietf/static/css/ietf.scss +++ b/ietf/static/css/ietf.scss @@ -1216,3 +1216,20 @@ iframe.status { .overflow-shadows--bottom-only { box-shadow: inset 0px -21px 18px -20px var(--bs-body-color); } + +#navbar-doc-search-wrapper { + position: relative; +} + +#navbar-doc-search-results { + max-height: 400px; + overflow-y: auto; + min-width: auto; + left: 0; + right: 0; + + .dropdown-item { + white-space: normal; + overflow-wrap: break-word; + } +} diff --git a/ietf/static/js/navbar-doc-search.js b/ietf/static/js/navbar-doc-search.js new file mode 100644 index 0000000000..c36c032310 --- /dev/null +++ b/ietf/static/js/navbar-doc-search.js @@ -0,0 +1,113 @@ +$(function () { + var $input = $('#navbar-doc-search'); + var $results = $('#navbar-doc-search-results'); + var ajaxUrl = $input.data('ajax-url'); + var debounceTimer = null; + var highlightedIndex = -1; + var keyboardHighlight = false; + var currentItems = []; + + function showDropdown() { + $results.addClass('show'); + } + + function hideDropdown() { + $results.removeClass('show'); + highlightedIndex = -1; + keyboardHighlight = false; + updateHighlight(); + } + + function updateHighlight() { + $results.find('.dropdown-item').removeClass('active'); + if (highlightedIndex >= 0 && highlightedIndex < currentItems.length) { + $results.find('.dropdown-item').eq(highlightedIndex).addClass('active'); + } + } + + function doSearch(query) { + if (query.length < 2) { + hideDropdown(); + return; + } + $.ajax({ + url: ajaxUrl, + dataType: 'json', + data: { q: query }, + success: function (data) { + currentItems = data; + highlightedIndex = -1; + $results.empty(); + if (data.length === 0) { + $results.append('
      • No results found
        • '); + } else { + data.forEach(function (item) { + var $li = $('
      • '); + var $a = $('' + item.text + ''); + $li.append($a); + $results.append($li); + }); + } + showDropdown(); + } + }); + } + + $input.on('input', function () { + clearTimeout(debounceTimer); + var query = $(this).val().trim(); + debounceTimer = setTimeout(function () { + doSearch(query); + }, 250); + }); + + $input.on('keydown', function (e) { + if (e.key === 'ArrowDown') { + e.preventDefault(); + if (highlightedIndex < currentItems.length - 1) { + highlightedIndex++; + keyboardHighlight = true; + updateHighlight(); + } + } else if (e.key === 'ArrowUp') { + e.preventDefault(); + if (highlightedIndex > 0) { + highlightedIndex--; + keyboardHighlight = true; + updateHighlight(); + } + } else if (e.key === 'Enter') { + e.preventDefault(); + if (keyboardHighlight && highlightedIndex >= 0 && highlightedIndex < currentItems.length) { + window.location.href = currentItems[highlightedIndex].url; + } else { + var query = $(this).val().trim(); + if (query) { + window.location.href = '/doc/search/?name=' + encodeURIComponent(query) + '&rfcs=on&activedrafts=on&olddrafts=on'; + } + } + } else if (e.key === 'Escape') { + hideDropdown(); + $input.blur(); + } + }); + + // Hover highlights (visual only — Enter still submits the text) + $results.on('mouseenter', '.dropdown-item', function () { + highlightedIndex = $results.find('.dropdown-item').index(this); + keyboardHighlight = false; + updateHighlight(); + }); + + $results.on('mouseleave', '.dropdown-item', function () { + highlightedIndex = -1; + updateHighlight(); + }); + + // Click outside closes dropdown + $(document).on('click', function (e) { + if (!$(e.target).closest('#navbar-doc-search-wrapper').length) { + hideDropdown(); + } + }); +}); diff --git a/ietf/templates/base.html b/ietf/templates/base.html index 25ce50c467..b0df04f30a 100644 --- a/ietf/templates/base.html +++ b/ietf/templates/base.html @@ -67,13 +67,17 @@ {% endif %} - +

        • +

        + The not-prepped XML + is the RFC XML v3 source for an RFC at the moment in the publication process + just before the prep tool was used to expand default + values, generate section numbers, resolve cross-references, and embed + boilerplate. +

        + It is useful for authors who want to begin a new draft based on + the RFC's text, such as when creating a bis-draft, and for tools that process + author-facing RFC XML. +

        +

        + + + Download not-prepped XML for RFC {{ rfc.rfc_number }} + +

        +{% endblock %} From 20480d6242254693b3dbcbf9b73380b5b3c838cb Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Apr 2026 13:59:23 -0500 Subject: [PATCH 152/158] fix: force notprepped downloads (#10719) --- ietf/doc/{tests_unprepped.py => tests_notprepped.py} | 10 +++++++--- ietf/doc/views_doc.py | 5 +++-- 2 files changed, 10 insertions(+), 5 deletions(-) rename ietf/doc/{tests_unprepped.py => tests_notprepped.py} (92%) diff --git a/ietf/doc/tests_unprepped.py b/ietf/doc/tests_notprepped.py similarity index 92% rename from ietf/doc/tests_unprepped.py rename to ietf/doc/tests_notprepped.py index f88af8e81a..f417aa7931 100644 --- a/ietf/doc/tests_unprepped.py +++ b/ietf/doc/tests_notprepped.py @@ -12,7 +12,7 @@ from ietf.utils.test_utils import TestCase -class UnpreppedRfcXmlTests(TestCase): +class NotpreppedRfcXmlTests(TestCase): def test_editor_source_button_visibility(self): pre_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC - 1) first_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC) @@ -72,13 +72,17 @@ def test_rfcxml_notprepped(self): r = self.client.get(url) self.assertEqual(r.status_code, 404) - # 200 with correct content-type and body when object is fully stored + # 200 with correct content-type, attachment disposition, and body when object is fully stored xml_content = b"test" store_bytes("rfc", stored_name, xml_content, allow_overwrite=True) r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertEqual(r["Content-Type"], "application/xml") - self.assertEqual(r.content, xml_content) + self.assertEqual( + r["Content-Disposition"], + f'attachment; filename="rfc{number}.notprepped.xml"', + ) + self.assertEqual(b"".join(r.streaming_content), xml_content) def test_rfcxml_notprepped_wrapper(self): number = settings.FIRST_V3_RFC diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index a23185333e..5b57a62074 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -43,9 +43,10 @@ from celery.result import AsyncResult from django.core.cache import caches +from django.core.files.base import ContentFile from django.core.exceptions import PermissionDenied from django.db.models import Max -from django.http import HttpResponse, Http404, HttpResponseBadRequest, JsonResponse +from django.http import FileResponse, HttpResponse, Http404, HttpResponseBadRequest, JsonResponse from django.shortcuts import render, get_object_or_404, redirect from django.template.loader import render_to_string from django.urls import reverse as urlreverse @@ -2372,7 +2373,7 @@ def rfcxml_notprepped(request, number): bytes = retrieve_bytes("rfc", name) except FileNotFoundError: raise Http404 - return HttpResponse(bytes, content_type="application/xml") + return FileResponse(ContentFile(bytes, name=f"rfc{number}.notprepped.xml"), as_attachment=True) def rfcxml_notprepped_wrapper(request, number): From 9cecc36bc7e42ecc5cd196d96f4bd0eaf03b5e69 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Apr 2026 00:25:02 -0300 Subject: [PATCH 153/158] feat: rebuild_searchindex task (#10723) * refactor: DRY * chore: typesense docker container (commented out) * feat: batched RFC search index import * feat: rebuild_searchindex task * feat: logging / error reporting * refactor: _task suffix for task name * test: tests for searchindex utils + tasks * fix: only create collection if dropped * fix: typing / lint --- docker-compose.yml | 12 ++ ietf/doc/tasks.py | 11 ++ ietf/doc/tests_tasks.py | 43 ++++++ ietf/utils/searchindex.py | 239 ++++++++++++++++++++++++++++++-- ietf/utils/tests_searchindex.py | 152 +++++++++++++++----- 5 files changed, 410 insertions(+), 47 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4c3f2f6b8e..073d04b896 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -132,6 +132,18 @@ services: volumes: - blobdb-data:/var/lib/postgresql/data +# typesense: +# image: typesense/typesense:30.1 +# restart: on-failure +# ports: +# - "8108:8108" +# volumes: +# - ./typesense-data:/data +# command: +# - '--data-dir=/data' +# - '--api-key=typesense-api-key' +# - '--enable-cors' + # Celery Beat is a periodic task runner. It is not normally needed for development, # but can be enabled by uncommenting the following. # diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index 19edb39014..273242e35f 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -209,3 +209,14 @@ def update_rfc_searchindex_task(self, rfc_number: int): countdown=searchindex_settings["TASK_RETRY_DELAY"], max_retries=searchindex_settings["TASK_MAX_RETRIES"], ) + + +@shared_task +def rebuild_searchindex_task(*, batchsize=40, drop_collection=False): + if drop_collection: + searchindex.delete_collection() + searchindex.create_collection() + searchindex.update_or_create_rfc_entries( + Document.objects.filter(type_id="rfc").order_by("-rfc_number"), + batchsize=batchsize, + ) diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 728d21f131..2e2d65463f 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -24,6 +24,7 @@ generate_idnits2_rfc_status_task, investigate_fragment_task, notify_expirations_task, + rebuild_searchindex_task, update_rfc_searchindex_task, ) @@ -144,6 +145,48 @@ def test_update_rfc_searchindex_task( with self.assertRaises(Retry): update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + @mock.patch("ietf.doc.tasks.searchindex.update_or_create_rfc_entries") + @mock.patch("ietf.doc.tasks.searchindex.create_collection") + @mock.patch("ietf.doc.tasks.searchindex.delete_collection") + def test_rebuild_searchindex_task(self, mock_delete, mock_create, mock_update): + rfcs = WgRfcFactory.create_batch(10) + rebuild_searchindex_task() + self.assertFalse(mock_delete.called) + self.assertFalse(mock_create.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + + mock_delete.reset_mock() + mock_create.reset_mock() + mock_update.reset_mock() + rebuild_searchindex_task(drop_collection=True) + self.assertTrue(mock_delete.called) + self.assertTrue(mock_create.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + + mock_delete.reset_mock() + mock_create.reset_mock() + mock_update.reset_mock() + rebuild_searchindex_task(drop_collection=True, batchsize=3) + self.assertTrue(mock_delete.called) + self.assertTrue(mock_create.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + self.assertEqual(mock_update.call_args.kwargs["batchsize"], 3) + class Idnits2SupportTests(TestCase): settings_temp_path_overrides = TestCase.settings_temp_path_overrides + [ diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py index e4427b88b5..a47e6d2f12 100644 --- a/ietf/utils/searchindex.py +++ b/ietf/utils/searchindex.py @@ -2,12 +2,15 @@ """Search indexing utilities""" import re +from itertools import batched from math import floor +from typing import Iterable import httpx # just for exceptions import typesense import typesense.exceptions from django.conf import settings +from typesense.types.document import DocumentSchema from ietf.doc.models import Document, StoredObject from ietf.doc.storage_utils import retrieve_str @@ -42,6 +45,24 @@ def enabled(): return _settings["TYPESENSE_API_URL"] != "" +def get_typesense_client() -> typesense.Client: + _settings = get_settings() + client = typesense.Client( + { + "api_key": _settings["TYPESENSE_API_KEY"], + "nodes": [_settings["TYPESENSE_API_URL"]], + } + ) + return client + + +def get_collection_name() -> str: + _settings = get_settings() + collection_name = _settings["TYPESENSE_COLLECTION_NAME"] + assert isinstance(collection_name, str) + return collection_name + + def _sanitize_text(content): """Sanitize content or abstract text for search""" # REs (with approximate names) @@ -62,7 +83,7 @@ def _sanitize_text(content): return content.strip() -def update_or_create_rfc_entry(rfc: Document): +def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: assert rfc.type_id == "rfc" assert rfc.rfc_number is not None @@ -75,8 +96,8 @@ def update_or_create_rfc_entry(rfc: Document): f"Indexing as {subseries[0].name}" ) subseries = subseries[0] if len(subseries) > 0 else None - obsoleted_by = rfc.relations_that("obs") - updated_by = rfc.relations_that("updates") + obsoleted_by = rfc.related_that("obs") + updated_by = rfc.related_that("updates") stored_txt = ( StoredObject.objects.exclude_deleted() @@ -91,8 +112,8 @@ def update_or_create_rfc_entry(rfc: Document): except Exception as err: log(f"Unable to retrieve {stored_txt} from storage: {err}") - ts_id = f"doc-{rfc.pk}" ts_document = { + "id": f"doc-{rfc.pk}", "rfcNumber": rfc.rfc_number, "rfc": str(rfc.rfc_number), "filename": rfc.name, @@ -143,13 +164,205 @@ def update_or_create_rfc_entry(rfc: Document): ts_document["adName"] = rfc.ad.name if content != "": ts_document["content"] = _sanitize_text(content) - _settings = get_settings() - client = typesense.Client( + return ts_document + + +def update_or_create_rfc_entry(rfc: Document): + """Update/create index entries for one RFC""" + ts_document = typesense_doc_from_rfc(rfc) + client = get_typesense_client() + client.collections[get_collection_name()].documents.upsert(ts_document) + + +def update_or_create_rfc_entries( + rfcs: Iterable[Document], batchsize: int | None = None +): + """Update/create index entries for RFCs in bulk + + If batchsize is set, computes index data in batches of batchsize and adds to the + index. Will make a total of (len(rfcs) // batchsize) + 1 API calls. + + N.b. that typesense has a server-side batch size that defaults to 40, which should + "almost never be changed from the default." This does not change that. Further, + the python client library's import_ method has a batch_size parameter that does + client-side batching. We don't use that, either. + """ + success_count = 0 + fail_count = 0 + client = get_typesense_client() + batches = [rfcs] if batchsize is None else batched(rfcs, batchsize) + for batch in batches: + tdoc_batch = [typesense_doc_from_rfc(rfc) for rfc in batch] + results = client.collections[get_collection_name()].documents.import_( + tdoc_batch, {"action": "upsert"} + ) + for tdoc, result in zip(tdoc_batch, results): + if result["success"]: + success_count += 1 + else: + fail_count += 1 + log(f"Failed to index RFC {tdoc['rfcNumber']}: {result['error']}") + log(f"Added {success_count} RFCs to the index, failed to add {fail_count}") + + +DOCS_SCHEMA = { + "enable_nested_fields": True, + "default_sorting_field": "ranking", + "fields": [ + # RFC number in integer form, for sorting asc/desc in search results + # Omit field for drafts { - "api_key": _settings["TYPESENSE_API_KEY"], - "nodes": [_settings["TYPESENSE_API_URL"]], - } - ) - client.collections[_settings["TYPESENSE_COLLECTION_NAME"]].documents.upsert( - {"id": ts_id} | ts_document - ) + "name": "rfcNumber", + "type": "int32", + "facet": False, + "optional": True, + "sort": True, + }, + # RFC number in string form, for direct matching with ranking + # Omit field for drafts + {"name": "rfc", "type": "string", "facet": False, "optional": True}, + # For drafts that correspond to an RFC, insert the RFC number + # Omit field for rfcs or if not relevant + {"name": "ref", "type": "string", "facet": False, "optional": True}, + # Filename of the document (without the extension, e.g. "rfc1234" + # or "draft-ietf-abc-def-02") + {"name": "filename", "type": "string", "facet": False, "infix": True}, + # Title of the draft / rfc + {"name": "title", "type": "string", "facet": False}, + # Abstract of the draft / rfc + {"name": "abstract", "type": "string", "facet": False}, + # A list of search keywords if relevant, set to empty array otherwise + {"name": "keywords", "type": "string[]", "facet": True}, + # Type of the document + # Accepted values: "draft" or "rfc" + {"name": "type", "type": "string", "facet": True}, + # State(s) of the document (e.g. "Published", "Adopted by a WG", etc.) + # Use the full name, not the slug + {"name": "state", "type": "string[]", "facet": True, "optional": True}, + # Status (Standard Level Name) + # Object with properties "slug" and "name" + # e.g.: { slug: "std", "name": "Internet Standard" } + {"name": "status", "type": "object", "facet": True, "optional": True}, + # The subseries it is part of. (e.g. "BCP") + # Omit otherwise. + { + "name": "subseries.acronym", + "type": "string", + "facet": True, + "optional": True, + }, + # The subseries number it is part of. (e.g. 123) + # Omit otherwise. + { + "name": "subseries.number", + "type": "int32", + "facet": True, + "sort": True, + "optional": True, + }, + # The total of RFCs in the subseries + # Omit if not part of a subseries + { + "name": "subseries.total", + "type": "int32", + "facet": False, + "sort": False, + "optional": True, + }, + # Date of the document, in unix epoch seconds (can be negative for < 1970) + {"name": "date", "type": "int64", "facet": False}, + # Expiration date of the document, in unix epoch seconds (can be negative + # for < 1970). Omit field for RFCs + {"name": "expires", "type": "int64", "facet": False, "optional": True}, + # Publication date of the RFC, in unix epoch seconds (can be negative + # for < 1970). Omit field for drafts + { + "name": "publicationDate", + "type": "int64", + "facet": True, + "optional": True, + }, + # Working Group + # Object with properties "acronym", "name" and "full" + # e.g.: + # { + # "acronym": "ntp", + # "name": "Network Time Protocols", + # "full": "ntp - Network Time Protocols", + # } + {"name": "group", "type": "object", "facet": True, "optional": True}, + # Area + # Object with properties "acronym", "name" and "full" + # e.g.: + # { + # "acronym": "mpls", + # "name": "Multiprotocol Label Switching", + # "full": "mpls - Multiprotocol Label Switching", + # } + {"name": "area", "type": "object", "facet": True, "optional": True}, + # Stream + # Object with properties "slug" and "name" + # e.g.: { slug: "ietf", "name": "IETF" } + {"name": "stream", "type": "object", "facet": True, "optional": True}, + # List of authors + # Array of objects with properties "name" and "affiliation" + # e.g.: + # [ + # {"name": "John Doe", "affiliation": "ACME Inc."}, + # {"name": "Ada Lovelace", "affiliation": "Babbage Corps."}, + # ] + {"name": "authors", "type": "object[]", "facet": True, "optional": True}, + # Area Director Name (e.g. "Leonardo DaVinci") + {"name": "adName", "type": "string", "facet": True, "optional": True}, + # Whether the document should be hidden by default in search results or not. + {"name": "flags.hiddenDefault", "type": "bool", "facet": True}, + # Whether the document is obsoleted by another document or not. + {"name": "flags.obsoleted", "type": "bool", "facet": True}, + # Whether the document is updated by another document or not. + {"name": "flags.updated", "type": "bool", "facet": True}, + # List of documents that obsolete this document. + # Array of strings. Use RFC number for RFCs. (e.g. ["123", "456"]) + # Omit if none. Must be provided if "flags.obsoleted" is set to True. + { + "name": "obsoletedBy", + "type": "string[]", + "facet": False, + "optional": True, + }, + # List of documents that update this document. + # Array of strings. Use RFC number for RFCs. (e.g. ["123", "456"]) + # Omit if none. Must be provided if "flags.updated" is set to True. + {"name": "updatedBy", "type": "string[]", "facet": False, "optional": True}, + # Sanitized content of the document. + # Make sure to remove newlines, double whitespaces, symbols and tags. + { + "name": "content", + "type": "string", + "facet": False, + "optional": True, + "store": False, + }, + # Ranking value to use when no explicit sorting is used during search + # Set to the RFC number for RFCs and the revision number for drafts + # This ensures newer RFCs get listed first in the default search results + # (without a query) + {"name": "ranking", "type": "int32", "facet": False}, + ], +} + + +def create_collection(): + collection_name = get_collection_name() + log(f"Creating '{collection_name}' collection") + client = get_typesense_client() + client.collections.create({"name": get_collection_name()} | DOCS_SCHEMA) + + +def delete_collection(): + collection_name = get_collection_name() + log(f"Deleting '{collection_name}' collection") + client = get_typesense_client() + try: + client.collections[collection_name].delete() + except typesense.exceptions.ObjectNotFound: + pass diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py index 8740716c85..0bff96ec7d 100644 --- a/ietf/utils/tests_searchindex.py +++ b/ietf/utils/tests_searchindex.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2026, All Rights Reserved from unittest import mock +import typesense.exceptions from django.conf import settings from django.test.utils import override_settings @@ -51,42 +52,29 @@ def test_sanitize_text(self): "TYPESENSE_COLLECTION_NAME": "frogs", } ) - @mock.patch("ietf.utils.searchindex.typesense.Client") - def test_update_or_create_rfc_entry(self, mock_ts_client_constructor): + def test_typesense_doc_from_rfc(self): not_rfc = WgDraftFactory() assert isinstance(not_rfc, Document) with self.assertRaises(AssertionError): - searchindex.update_or_create_rfc_entry(not_rfc) - self.assertFalse(mock_ts_client_constructor.called) + searchindex.typesense_doc_from_rfc(not_rfc) invalid_rfc = WgRfcFactory(name="rfc1000000", rfc_number=None) assert isinstance(invalid_rfc, Document) with self.assertRaises(AssertionError): - searchindex.update_or_create_rfc_entry(invalid_rfc) - self.assertFalse(mock_ts_client_constructor.called) + searchindex.typesense_doc_from_rfc(invalid_rfc) rfc = PublishedRfcDocEventFactory().doc assert isinstance(rfc, Document) - searchindex.update_or_create_rfc_entry(rfc) - self.assertTrue(mock_ts_client_constructor.called) - # walk the tree down to the method we expected to be called... - mock_upsert = mock_ts_client_constructor.return_value.collections[ - "frogs" - ].documents.upsert # matches value in override_settings above - self.assertTrue(mock_upsert.called) - upserted_dict = mock_upsert.call_args[0][0] + result = searchindex.typesense_doc_from_rfc(rfc) # Check a few values, not exhaustive - self.assertEqual(upserted_dict["id"], f"doc-{rfc.pk}") - self.assertEqual(upserted_dict["rfcNumber"], rfc.rfc_number) - self.assertEqual( - upserted_dict["abstract"], searchindex._sanitize_text(rfc.abstract) - ) - self.assertNotIn("adName", upserted_dict) - self.assertNotIn("content", upserted_dict) # no blob - self.assertNotIn("subseries", upserted_dict) + self.assertEqual(result["id"], f"doc-{rfc.pk}") + self.assertEqual(result["rfcNumber"], rfc.rfc_number) + self.assertEqual(result["abstract"], searchindex._sanitize_text(rfc.abstract)) + self.assertNotIn("adName", result) + self.assertNotIn("content", result) # no blob + self.assertNotIn("subseries", result) # repeat, this time with contents, an AD, and subseries docs - mock_upsert.reset_mock() store_str( kind="rfc", name=f"txt/{rfc.name}.txt", @@ -99,17 +87,15 @@ def test_update_or_create_rfc_entry(self, mock_ts_client_constructor): # (the typesense schema does not support this for real at the moment) BcpFactory(contains=[rfc], name="bcp1234") StdFactory(contains=[rfc], name="std1234") - searchindex.update_or_create_rfc_entry(rfc) - self.assertTrue(mock_upsert.called) - upserted_dict = mock_upsert.call_args[0][0] + result = searchindex.typesense_doc_from_rfc(rfc) # Check a few values, not exhaustive self.assertEqual( - upserted_dict["content"], + result["content"], searchindex._sanitize_text("The contents of this RFC"), ) - self.assertEqual(upserted_dict["adName"], "Alfred D. Rector") - self.assertIn("subseries", upserted_dict) - ss_dict = upserted_dict["subseries"] + self.assertEqual(result["adName"], "Alfred D. Rector") + self.assertIn("subseries", result) + ss_dict = result["subseries"] # We should get one of the two subseries docs, but neither is more correct # than the other... self.assertTrue( @@ -119,10 +105,108 @@ def test_update_or_create_rfc_entry(self, mock_ts_client_constructor): ) ) - # Finally, delete the contents blob and make sure things don't blow up - mock_upsert.reset_mock() + # Finally, delete the contents blob and make sure things don't blow up Blob.objects.get(bucket="rfc", name=f"txt/{rfc.name}.txt").delete() + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertNotIn("content", result) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense_doc_from_rfc") + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_update_or_create_rfc_entry( + self, mock_ts_client_constructor, mock_tdoc_from_rfc + ): + fake_tdoc = object() + mock_tdoc_from_rfc.return_value = fake_tdoc + rfc = WgRfcFactory() + assert isinstance(rfc, Document) searchindex.update_or_create_rfc_entry(rfc) + self.assertTrue(mock_ts_client_constructor.called) + # walk the tree down to the method we expected to be called... + mock_upsert = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.upsert self.assertTrue(mock_upsert.called) - upserted_dict = mock_upsert.call_args[0][0] - self.assertNotIn("content", upserted_dict) + self.assertEqual(mock_upsert.call_args, mock.call(fake_tdoc)) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense_doc_from_rfc") + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_update_or_create_rfc_entries( + self, mock_ts_client_constructor, mock_tdoc_from_rfc + ): + fake_tdoc = object() + mock_tdoc_from_rfc.return_value = fake_tdoc + rfc = WgRfcFactory() + assert isinstance(rfc, Document) + searchindex.update_or_create_rfc_entries([rfc] * 50) # list of docs... + self.assertEqual(mock_ts_client_constructor.call_count, 1) + # walk the tree down to the method we expected to be called... + mock_import_ = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.import_ + self.assertEqual(mock_import_.call_count, 1) + self.assertEqual( + mock_import_.call_args, mock.call([fake_tdoc] * 50, {"action": "upsert"}) + ) + + mock_import_.reset_mock() + searchindex.update_or_create_rfc_entries([rfc] * 50, batchsize=20) + self.assertEqual(mock_ts_client_constructor.call_count, 2) # one more + # walk the tree down to the method we expected to be called... + mock_import_ = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.import_ + self.assertEqual(mock_import_.call_count, 3) + self.assertEqual( + mock_import_.call_args_list, + [ + mock.call([fake_tdoc] * 20, {"action": "upsert"}), + mock.call([fake_tdoc] * 20, {"action": "upsert"}), + mock.call([fake_tdoc] * 10, {"action": "upsert"}), + ], + ) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_create_collection(self, mock_ts_client_constructor): + searchindex.create_collection() + self.assertEqual(mock_ts_client_constructor.call_count, 1) + mock_collections = mock_ts_client_constructor.return_value.collections + self.assertTrue(mock_collections.create.called) + self.assertEqual(mock_collections.create.call_args[0][0]["name"], "frogs") + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_delete_collection(self, mock_ts_client_constructor): + searchindex.delete_collection() + self.assertEqual(mock_ts_client_constructor.call_count, 1) + mock_collections = mock_ts_client_constructor.return_value.collections + self.assertTrue(mock_collections["frogs"].delete.called) + + mock_collections["frogs"].side_effect = typesense.exceptions.ObjectNotFound + searchindex.delete_collection() # should ignore the exception From c4cb8b91fc9434a3bb3419acfac2dd3b30cb4a6c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Apr 2026 07:33:51 -0300 Subject: [PATCH 154/158] fix: add pages to typesense schema (#10726) --- ietf/utils/searchindex.py | 4 ++++ ietf/utils/tests_searchindex.py | 1 + 2 files changed, 5 insertions(+) diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py index a47e6d2f12..87951abb60 100644 --- a/ietf/utils/searchindex.py +++ b/ietf/utils/searchindex.py @@ -86,6 +86,7 @@ def _sanitize_text(content): def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: assert rfc.type_id == "rfc" assert rfc.rfc_number is not None + assert rfc.pages is not None keywords: list[str] = rfc.keywords # help type checking @@ -119,6 +120,7 @@ def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: "filename": rfc.name, "title": rfc.title, "abstract": _sanitize_text(rfc.abstract), + "pages": rfc.pages, "keywords": keywords, "type": "rfc", "state": [state.name for state in rfc.states.all()], @@ -231,6 +233,8 @@ def update_or_create_rfc_entries( {"name": "title", "type": "string", "facet": False}, # Abstract of the draft / rfc {"name": "abstract", "type": "string", "facet": False}, + # Number of pages + {"name": "pages", "type": "int32", "facet": False}, # A list of search keywords if relevant, set to empty array otherwise {"name": "keywords", "type": "string[]", "facet": True}, # Type of the document diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py index 0bff96ec7d..e9fbf52020 100644 --- a/ietf/utils/tests_searchindex.py +++ b/ietf/utils/tests_searchindex.py @@ -70,6 +70,7 @@ def test_typesense_doc_from_rfc(self): self.assertEqual(result["id"], f"doc-{rfc.pk}") self.assertEqual(result["rfcNumber"], rfc.rfc_number) self.assertEqual(result["abstract"], searchindex._sanitize_text(rfc.abstract)) + self.assertEqual(result["pages"], rfc.pages) self.assertNotIn("adName", result) self.assertNotIn("content", result) # no blob self.assertNotIn("subseries", result) From 629ffb13480201e25fc5d941cfcea9de123562f9 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Apr 2026 15:04:23 -0300 Subject: [PATCH 155/158] fix: decode non-utf-8 blob content (#10729) * refactor: decode_document_content() utility method * fix: fall back to latin-1 in retrieve_str() * refactor: match structure with retrieve_bytes() * refactor: separate tests_text.py module * test: test_decode_document_content + ruff * fix: revert misguided refactor * test: assert to guarantee test is valid --- ietf/doc/models.py | 15 ++------- ietf/doc/storage_utils.py | 47 +++++++++++++------------- ietf/utils/tests.py | 19 ----------- ietf/utils/tests_text.py | 71 +++++++++++++++++++++++++++++++++++++++ ietf/utils/text.py | 18 ++++++++++ 5 files changed, 114 insertions(+), 56 deletions(-) create mode 100644 ietf/utils/tests_text.py diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 972f0a34e8..cc79b73831 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -52,6 +52,7 @@ from ietf.person.utils import get_active_balloters from ietf.utils import log from ietf.utils.decorators import memoize +from ietf.utils.text import decode_document_content from ietf.utils.validators import validate_no_control_chars from ietf.utils.mail import formataddr from ietf.utils.models import ForeignKey @@ -640,19 +641,7 @@ def text(self, size = -1): except IOError as e: log.log(f"Error reading text for {path}: {e}") return None - text = None - try: - text = raw.decode('utf-8') - except UnicodeDecodeError: - for back in range(1,4): - try: - text = raw[:-back].decode('utf-8') - break - except UnicodeDecodeError: - pass - if text is None: - text = raw.decode('latin-1') - return text + return decode_document_content(raw) def text_or_error(self): return self.text() or "Error; cannot read '%s'"%self.get_base_name() diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index ffdd4599be..9c18bb8a8a 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -10,6 +10,7 @@ from django.core.files.storage import storages, Storage from ietf.utils.log import log +from ietf.utils.text import decode_document_content class StorageUtilsError(Exception): @@ -164,32 +165,30 @@ def store_str( def retrieve_bytes(kind: str, name: str) -> bytes: from ietf.doc.storage import maybe_log_timing - content = b"" - if settings.ENABLE_BLOBSTORAGE: - try: - store = _get_storage(kind) - with store.open(name) as f: - with maybe_log_timing( - hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing, - "read", - bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "", - name=name, - ): - content = f.read() - except Exception as err: - log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}") - raise + if not settings.ENABLE_BLOBSTORAGE: + return b"" + try: + store = _get_storage(kind) + with store.open(name) as f: + with maybe_log_timing( + hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing, + "read", + bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "", + name=name, + ): + content = f.read() + except Exception as err: + log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}") + raise return content def retrieve_str(kind: str, name: str) -> str: - content = "" - if settings.ENABLE_BLOBSTORAGE: - try: - content_bytes = retrieve_bytes(kind, name) - # TODO-BLOBSTORE: try to decode all the different ways doc.text() does - content = content_bytes.decode("utf-8") - except Exception as err: - log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}") - raise + if not settings.ENABLE_BLOBSTORAGE: + return "" + try: + content = decode_document_content(retrieve_bytes(kind, name)) + except Exception as err: + log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}") + raise return content diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 3288309095..99c33f34b3 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -60,7 +60,6 @@ set_url_coverage, ) from ietf.utils.test_utils import TestCase, unicontent -from ietf.utils.text import parse_unicode from ietf.utils.timezone import timezone_not_near_midnight from ietf.utils.xmldraft import XMLDraft, InvalidMetadataError, capture_xml2rfc_output @@ -864,24 +863,6 @@ def test_assertion(self): assertion('False') settings.SERVER_MODE = 'test' -class TestRFC2047Strings(TestCase): - def test_parse_unicode(self): - names = ( - ('=?utf-8?b?4Yuz4YuK4Ym1IOGJoOGJgOGIiA==?=', 'ዳዊት በቀለ'), - ('=?utf-8?b?5Li9IOmDnA==?=', '丽 郜'), - ('=?utf-8?b?4KSV4KSu4KWN4KSs4KWL4KScIOCkoeCkvuCksA==?=', 'कम्बोज डार'), - ('=?utf-8?b?zpfPgc6szrrOu861zrnOsSDOm865z4zOvc+Ezrc=?=', 'Ηράκλεια Λιόντη'), - ('=?utf-8?b?15nXqdeo15DXnCDXqNeV15bXoNek15zXkw==?=', 'ישראל רוזנפלד'), - ('=?utf-8?b?5Li95Y2OIOeahw==?=', '丽华 皇'), - ('=?utf-8?b?77ul77qu766V77qzIO+tlu+7ru+vvu+6ju+7pw==?=', 'ﻥﺮﮕﺳ ﭖﻮﯾﺎﻧ'), - ('=?utf-8?b?77uh77uu77qz77uu76++IO+6su+7tO+7p++6jSDvurDvu6Pvuo7vu6jvr74=?=', 'ﻡﻮﺳﻮﯾ ﺲﻴﻧﺍ ﺰﻣﺎﻨﯾ'), - ('=?utf-8?b?ScOxaWdvIFNhbsOnIEliw6HDsWV6IGRlIGxhIFBlw7Fh?=', 'Iñigo Sanç Ibáñez de la Peña'), - ('Mart van Oostendorp', 'Mart van Oostendorp'), - ('', ''), - ) - for encoded_str, unicode in names: - self.assertEqual(unicode, parse_unicode(encoded_str)) - class TestAndroidSiteManifest(TestCase): def test_manifest(self): r = self.client.get(urlreverse('site.webmanifest')) diff --git a/ietf/utils/tests_text.py b/ietf/utils/tests_text.py new file mode 100644 index 0000000000..51aa2eff13 --- /dev/null +++ b/ietf/utils/tests_text.py @@ -0,0 +1,71 @@ +# Copyright The IETF Trust 2021-2026, All Rights Reserved +from ietf.utils.test_utils import TestCase +from ietf.utils.text import parse_unicode, decode_document_content + + +class TestDecoders(TestCase): + def test_parse_unicode(self): + names = ( + ("=?utf-8?b?4Yuz4YuK4Ym1IOGJoOGJgOGIiA==?=", "ዳዊት በቀለ"), + ("=?utf-8?b?5Li9IOmDnA==?=", "丽 郜"), + ("=?utf-8?b?4KSV4KSu4KWN4KSs4KWL4KScIOCkoeCkvuCksA==?=", "कम्बोज डार"), + ("=?utf-8?b?zpfPgc6szrrOu861zrnOsSDOm865z4zOvc+Ezrc=?=", "Ηράκλεια Λιόντη"), + ("=?utf-8?b?15nXqdeo15DXnCDXqNeV15bXoNek15zXkw==?=", "ישראל רוזנפלד"), + ("=?utf-8?b?5Li95Y2OIOeahw==?=", "丽华 皇"), + ("=?utf-8?b?77ul77qu766V77qzIO+tlu+7ru+vvu+6ju+7pw==?=", "ﻥﺮﮕﺳ ﭖﻮﯾﺎﻧ"), + ( + "=?utf-8?b?77uh77uu77qz77uu76++IO+6su+7tO+7p++6jSDvurDvu6Pvuo7vu6jvr74=?=", + "ﻡﻮﺳﻮﯾ ﺲﻴﻧﺍ ﺰﻣﺎﻨﯾ", + ), + ( + "=?utf-8?b?ScOxaWdvIFNhbsOnIEliw6HDsWV6IGRlIGxhIFBlw7Fh?=", + "Iñigo Sanç Ibáñez de la Peña", + ), + ("Mart van Oostendorp", "Mart van Oostendorp"), + ("", ""), + ) + for encoded_str, unicode in names: + self.assertEqual(unicode, parse_unicode(encoded_str)) + + def test_decode_document_content(self): + utf8_bytes = "𒀭𒊩𒌆𒄈𒋢".encode("utf-8") # ends with 4-byte character + latin1_bytes = "àéîøü".encode("latin-1") + other_bytes = "àéîøü".encode("macintosh") # different from its latin-1 encoding + assert other_bytes.decode("macintosh") != other_bytes.decode("latin-1"),\ + "test broken: other_bytes must decode differently as latin-1" + + # simplest case + self.assertEqual( + decode_document_content(utf8_bytes), + utf8_bytes.decode(), + ) + # losing 1-4 bytes from the end leave the last character incomplete; the + # decoder should decode all but that last character + self.assertEqual( + decode_document_content(utf8_bytes[:-1]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-2]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-3]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-4]), + utf8_bytes.decode()[:-1], + ) + + # latin-1 is also simple + self.assertEqual( + decode_document_content(latin1_bytes), + latin1_bytes.decode("latin-1"), + ) + + # other character sets are just treated as latin1 (bug? feature? you decide) + self.assertEqual( + decode_document_content(other_bytes), + other_bytes.decode("latin-1"), + ) diff --git a/ietf/utils/text.py b/ietf/utils/text.py index 590ec3fd30..2763056e1a 100644 --- a/ietf/utils/text.py +++ b/ietf/utils/text.py @@ -263,3 +263,21 @@ def parse_unicode(text): else: text = decoded_string return text + + +def decode_document_content(content: bytes) -> str: + """Decode document contents as utf-8 or latin1 + + Method was developed in DocumentInfo.text() where it gave acceptable results + for existing documents / RFCs. + """ + try: + return content.decode("utf-8") + except UnicodeDecodeError: + pass + for back in range(1, 4): + try: + return content[:-back].decode("utf-8") + except UnicodeDecodeError: + pass + return content.decode("latin-1") # everything is legal in latin-1 From 63a69945ab11b1c3b3ec490fb260073c90eed0bc Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 17 Apr 2026 16:24:18 -0500 Subject: [PATCH 156/158] test: Squash some transient test error vectors (#10730) * test: enforce queryset order assumed by test * test: match html escaping in test * test: search more specifically for tokens to avoid mis-reading them when they occur in faker data --- ietf/group/tests_review.py | 30 +++++++++++++------------- ietf/meeting/tests_session_requests.py | 2 +- ietf/meeting/tests_views.py | 7 +++--- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/ietf/group/tests_review.py b/ietf/group/tests_review.py index 89c755bb26..bb9b79a416 100644 --- a/ietf/group/tests_review.py +++ b/ietf/group/tests_review.py @@ -888,10 +888,10 @@ def test_requests_history_filter_page(self): self.assertEqual(r.status_code, 200) self.assertContains(r, review_req.doc.name) self.assertContains(r, review_req2.doc.name) - self.assertContains(r, 'Assigned') - self.assertContains(r, 'Accepted') - self.assertContains(r, 'Completed') - self.assertContains(r, 'Ready') + self.assertContains(r, 'data-text="Assigned"') + self.assertContains(r, 'data-text="Accepted"') + self.assertContains(r, 'data-text="Completed"') + self.assertContains(r, 'data-text="Ready"') self.assertContains(r, escape(assignment.reviewer.person.name)) self.assertContains(r, escape(assignment2.reviewer.person.name)) @@ -907,10 +907,10 @@ def test_requests_history_filter_page(self): self.assertEqual(r.status_code, 200) self.assertContains(r, review_req.doc.name) self.assertNotContains(r, review_req2.doc.name) - self.assertContains(r, 'Assigned') - self.assertNotContains(r, 'Accepted') - self.assertNotContains(r, 'Completed') - self.assertNotContains(r, 'Ready') + self.assertContains(r, 'data-text="Assigned"') + self.assertNotContains(r, 'data-text="Accepted"') + self.assertNotContains(r, 'data-text="Completed"') + self.assertNotContains(r, 'data-text="Ready"') self.assertContains(r, escape(assignment.reviewer.person.name)) self.assertNotContains(r, escape(assignment2.reviewer.person.name)) @@ -926,10 +926,10 @@ def test_requests_history_filter_page(self): self.assertEqual(r.status_code, 200) self.assertNotContains(r, review_req.doc.name) self.assertContains(r, review_req2.doc.name) - self.assertNotContains(r, 'Assigned') - self.assertContains(r, 'Accepted') - self.assertContains(r, 'Completed') - self.assertContains(r, 'Ready') + self.assertNotContains(r, 'data-text="Assigned"') + self.assertContains(r, 'data-text="Accepted"') + self.assertContains(r, 'data-text="Completed"') + self.assertContains(r, 'data-text="Ready"') self.assertNotContains(r, escape(assignment.reviewer.person.name)) self.assertContains(r, escape(assignment2.reviewer.person.name)) @@ -940,9 +940,9 @@ def test_requests_history_filter_page(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertNotContains(r, review_req.doc.name) - self.assertNotContains(r, 'Assigned') - self.assertNotContains(r, 'Accepted') - self.assertNotContains(r, 'Completed') + self.assertNotContains(r, 'data-text="Assigned"') + self.assertNotContains(r, 'data-text="Accepted"') + self.assertNotContains(r, 'data-text="Completed"') def test_requests_history_invalid_filter_parameters(self): # First assignment as assigned diff --git a/ietf/meeting/tests_session_requests.py b/ietf/meeting/tests_session_requests.py index 0cb092d2f8..42dbee5f23 100644 --- a/ietf/meeting/tests_session_requests.py +++ b/ietf/meeting/tests_session_requests.py @@ -236,7 +236,7 @@ def test_edit(self): self.assertRedirects(r, redirect_url) # Check whether updates were stored in the database - sessions = Session.objects.filter(meeting=meeting, group=mars) + sessions = Session.objects.filter(meeting=meeting, group=mars).order_by("id") self.assertEqual(len(sessions), 2) session = sessions[0] self.assertFalse(session.constraints().filter(name='time_relation')) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 258ffe554c..17988e50be 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -33,6 +33,7 @@ from django.http import QueryDict, FileResponse from django.template import Context, Template from django.utils import timezone +from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.text import slugify @@ -9491,7 +9492,7 @@ def test_session_attendance(self): self.assertEqual(r.status_code, 200) self.assertContains(r, '3 attendees') for person in persons: - self.assertContains(r, person.plain_name()) + self.assertContains(r, escape(person.plain_name())) # Test for the "I was there" button. def _test_button(person, expected): @@ -9511,14 +9512,14 @@ def _test_button(person, expected): # attempt to POST anyway is ignored r = self.client.post(attendance_url) self.assertEqual(r.status_code, 200) - self.assertNotContains(r, persons[3].plain_name()) + self.assertNotContains(r, escape(persons[3].plain_name())) self.assertEqual(session.attended_set.count(), 3) # button is shown, and POST is accepted meeting.importantdate_set.update(name_id='revsub',date=date_today() + datetime.timedelta(days=20)) _test_button(persons[3], True) r = self.client.post(attendance_url) self.assertEqual(r.status_code, 200) - self.assertContains(r, persons[3].plain_name()) + self.assertContains(r, escape(persons[3].plain_name())) self.assertEqual(session.attended_set.count(), 4) # When the meeting is finalized, a bluesheet file is generated, From dc49dc8362812893cad560feecc55efcea1553dc Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 20 Apr 2026 14:29:41 -0300 Subject: [PATCH 157/158] chore: beat termination grace period -> 10 s (#10741) --- k8s/beat.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/beat.yaml b/k8s/beat.yaml index 9ab242681c..b4291c7e31 100644 --- a/k8s/beat.yaml +++ b/k8s/beat.yaml @@ -59,4 +59,4 @@ spec: name: files-cfgmap dnsPolicy: ClusterFirst restartPolicy: Always - terminationGracePeriodSeconds: 600 + terminationGracePeriodSeconds: 10 From 4d69329ef86054fa5bfb9da9acd0c966ab013d8f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 20 Apr 2026 23:59:36 -0300 Subject: [PATCH 158/158] chore: remove blobdb profiling logs (#10732) These are not useful any more, blobdb is fast --- ietf/doc/storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ietf/doc/storage.py b/ietf/doc/storage.py index 375620ccaf..ee1e76c4fa 100644 --- a/ietf/doc/storage.py +++ b/ietf/doc/storage.py @@ -114,7 +114,6 @@ def _get_write_parameters(self, name, content=None): class StoredObjectBlobdbStorage(BlobdbStorage): - ietf_log_blob_timing = True warn_if_missing = True # TODO-BLOBSTORE make this configurable (or remove it) def _save_stored_object(self, name, content) -> StoredObject: