From 15b2abd937f5d6e2b71667501309a965fdd9dc9d Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Apr 2026 12:12:58 -0500 Subject: [PATCH 01/17] feat: serve notprepped rfc xml from datatracker (#10692) * chore: move v3 RFC boundary number to settings * feat: button for downloading unprepped RFCXML * chore: ruff * chore: refactor test to avoid setup overhead * feat: serve notprepped bytes from blobdb * fix: typo * chore; improve test class name * feat: wrapper explaining notprepped xml --- ietf/doc/feeds.py | 3 +- ietf/doc/tests_unprepped.py | 118 +++++++++++++++++++++ ietf/doc/urls.py | 2 + ietf/doc/views_doc.py | 31 +++++- ietf/settings.py | 4 +- ietf/sync/rfcindex.py | 2 +- ietf/templates/doc/document_rfc.html | 9 ++ ietf/templates/doc/notprepped_wrapper.html | 26 +++++ 8 files changed, 190 insertions(+), 5 deletions(-) create mode 100644 ietf/doc/tests_unprepped.py create mode 100644 ietf/templates/doc/notprepped_wrapper.html diff --git a/ietf/doc/feeds.py b/ietf/doc/feeds.py index afe96cf0df..0269906fcf 100644 --- a/ietf/doc/feeds.py +++ b/ietf/doc/feeds.py @@ -5,6 +5,7 @@ import datetime import unicodedata +from django.conf import settings from django.contrib.syndication.views import Feed, FeedDoesNotExist from django.utils.feedgenerator import Atom1Feed, Rss201rev2Feed from django.urls import reverse as urlreverse @@ -223,7 +224,7 @@ def item_extra_kwargs(self, item): extra.update({"dcterms_accessRights": "gratis"}) extra.update({"dcterms_format": "text/html"}) media_contents = [] - if item.rfc_number < 8650: + if item.rfc_number < settings.FIRST_V3_RFC: if item.rfc_number not in [8, 9, 51, 418, 500, 530, 589]: for fmt, media_type in [("txt", "text/plain"), ("html", "text/html")]: media_contents.append( diff --git a/ietf/doc/tests_unprepped.py b/ietf/doc/tests_unprepped.py new file mode 100644 index 0000000000..f88af8e81a --- /dev/null +++ b/ietf/doc/tests_unprepped.py @@ -0,0 +1,118 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.conf import settings +from django.utils import timezone +from django.urls import reverse as urlreverse + +from pyquery import PyQuery + +from ietf.doc.factories import WgRfcFactory +from ietf.doc.models import StoredObject +from ietf.doc.storage_utils import store_bytes +from ietf.utils.test_utils import TestCase + + +class UnpreppedRfcXmlTests(TestCase): + def test_editor_source_button_visibility(self): + pre_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC - 1) + first_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC) + post_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC + 1) + + for rfc, expect_button in [(pre_v3, False), (first_v3, True), (post_v3, True)]: + r = self.client.get( + urlreverse( + "ietf.doc.views_doc.document_main", kwargs=dict(name=rfc.name) + ) + ) + self.assertEqual(r.status_code, 200) + buttons = PyQuery(r.content)('a.btn:contains("Get editor source")') + if expect_button: + self.assertEqual(len(buttons), 1, msg=f"rfc_number={rfc.rfc_number}") + expected_href = urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped_wrapper", + kwargs=dict(number=rfc.rfc_number), + ) + self.assertEqual( + buttons.attr("href"), + expected_href, + msg=f"rfc_number={rfc.rfc_number}", + ) + else: + self.assertEqual(len(buttons), 0, msg=f"rfc_number={rfc.rfc_number}") + + def test_rfcxml_notprepped(self): + number = settings.FIRST_V3_RFC + stored_name = f"notprepped/rfc{number}.notprepped.xml" + url = f"/doc/rfc{number}/notprepped/" + + # 404 for pre-v3 RFC numbers (no document needed) + r = self.client.get(f"/doc/rfc{number - 1}/notprepped/") + self.assertEqual(r.status_code, 404) + + # 404 when no RFC document exists in the database + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + # 404 when RFC document exists but has no StoredObject + WgRfcFactory(rfc_number=number) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + # 404 when StoredObject exists but backing storage is missing (FileNotFoundError) + now = timezone.now() + StoredObject.objects.create( + store="rfc", + name=stored_name, + sha384="a" * 96, + len=0, + store_created=now, + created=now, + modified=now, + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + # 200 with correct content-type and body when object is fully stored + xml_content = b"test" + store_bytes("rfc", stored_name, xml_content, allow_overwrite=True) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "application/xml") + self.assertEqual(r.content, xml_content) + + def test_rfcxml_notprepped_wrapper(self): + number = settings.FIRST_V3_RFC + + # 404 for pre-v3 RFC numbers (no document needed) + r = self.client.get( + urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped_wrapper", + kwargs=dict(number=number - 1), + ) + ) + self.assertEqual(r.status_code, 404) + + # 404 when no RFC document exists in the database + r = self.client.get( + urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped_wrapper", + kwargs=dict(number=number), + ) + ) + self.assertEqual(r.status_code, 404) + + # 200 with rendered template when RFC document exists + rfc = WgRfcFactory(rfc_number=number) + r = self.client.get( + urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped_wrapper", + kwargs=dict(number=number), + ) + ) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertIn(str(rfc.rfc_number), q("h1").text()) + download_url = urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped", kwargs=dict(number=number) + ) + self.assertEqual(len(q(f'a.btn[href="{download_url}"]')), 1) diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py index 61e94b2231..0c13503b78 100644 --- a/ietf/doc/urls.py +++ b/ietf/doc/urls.py @@ -99,6 +99,8 @@ url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main), url(r'^%(name)s(?:/%(rev)s)?/bibtex/$' % settings.URL_REGEXPS, views_doc.document_bibtex), + url(r'^rfc(?P[0-9]+)/notprepped/$' , views_doc.rfcxml_notprepped), + url(r'^rfc(?P[0-9]+)/notprepped-wrapper/$', views_doc.rfcxml_notprepped_wrapper), url(r'^%(name)s(?:/%(rev)s)?/idnits2-state/$' % settings.URL_REGEXPS, views_doc.idnits2_state), url(r'^bibxml3/reference.I-D.%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml_ref), url(r'^bibxml3/%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml), diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index c1f6352ac3..a23185333e 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2009-2024, All Rights Reserved +# Copyright The IETF Trust 2009-2026, All Rights Reserved # -*- coding: utf-8 -*- # # Parts Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies). @@ -57,7 +57,7 @@ import debug # pyflakes:ignore from ietf.doc.models import ( Document, DocHistory, DocEvent, BallotDocEvent, BallotType, - ConsensusDocEvent, NewRevisionDocEvent, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent, + ConsensusDocEvent, NewRevisionDocEvent, StoredObject, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent, IESG_BALLOT_ACTIVE_STATES, STATUSCHANGE_RELATIONS, DocumentActionHolder, DocumentAuthor, RelatedDocument, RelatedDocHistory) from ietf.doc.tasks import investigate_fragment_task @@ -86,6 +86,7 @@ from ietf.review.models import ReviewAssignment from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs, review_requests_to_list_for_docs from ietf.review.utils import no_review_from_teams_on_doc +from ietf.doc.storage_utils import retrieve_bytes from ietf.utils import markup_txt, log, markdown from ietf.utils.draft import get_status_from_draft_text from ietf.utils.meetecho import MeetechoAPIError, SlidesManager @@ -2356,3 +2357,29 @@ def investigate(request): "results": results, }, ) + +def rfcxml_notprepped(request, number): + number = int(number) + if number < settings.FIRST_V3_RFC: + raise Http404 + rfc = Document.objects.filter(type="rfc", rfc_number=number).first() + if rfc is None: + raise Http404 + name = f"notprepped/rfc{number}.notprepped.xml" + if not StoredObject.objects.filter(name=name).exists(): + raise Http404 + try: + bytes = retrieve_bytes("rfc", name) + except FileNotFoundError: + raise Http404 + return HttpResponse(bytes, content_type="application/xml") + + +def rfcxml_notprepped_wrapper(request, number): + number = int(number) + if number < settings.FIRST_V3_RFC: + raise Http404 + rfc = Document.objects.filter(type="rfc", rfc_number=number).first() + if rfc is None: + raise Http404 + return render(request, "doc/notprepped_wrapper.html", context={"rfc": rfc}) diff --git a/ietf/settings.py b/ietf/settings.py index 40a4cb5c56..3aa45a453c 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -235,7 +235,9 @@ AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', ) -FILE_UPLOAD_PERMISSIONS = 0o644 +FILE_UPLOAD_PERMISSIONS = 0o644 + +FIRST_V3_RFC = 8650 # diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index 6864617874..d1a0ed432f 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -153,7 +153,7 @@ def get_publication_std_levels() -> dict[int, StdLevelName]: def format_ordering(rfc_number): - if rfc_number < 8650: + if rfc_number < settings.FIRST_V3_RFC: ordering = ["txt", "ps", "pdf", "html", "xml"] else: ordering = ["html", "txt", "ps", "pdf", "xml"] diff --git a/ietf/templates/doc/document_rfc.html b/ietf/templates/doc/document_rfc.html index 7612ef8910..d4b309a964 100644 --- a/ietf/templates/doc/document_rfc.html +++ b/ietf/templates/doc/document_rfc.html @@ -124,6 +124,15 @@ Referenced by + {% if doc.rfc_number >= settings.FIRST_V3_RFC %} + + + + Get editor source + + {% endif %} RFC {{ rfc.rfc_number }} — Not-prepped XML +

+ The not-prepped XML + is the RFC XML v3 source for an RFC at the moment in the publication process + just before the prep tool was used to expand default + values, generate section numbers, resolve cross-references, and embed + boilerplate. +

+ It is useful for authors who want to begin a new draft based on + the RFC's text, such as when creating a bis-draft, and for tools that process + author-facing RFC XML. +

+

+ + + Download not-prepped XML for RFC {{ rfc.rfc_number }} + +

+{% endblock %} From 20480d6242254693b3dbcbf9b73380b5b3c838cb Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Apr 2026 13:59:23 -0500 Subject: [PATCH 02/17] fix: force notprepped downloads (#10719) --- ietf/doc/{tests_unprepped.py => tests_notprepped.py} | 10 +++++++--- ietf/doc/views_doc.py | 5 +++-- 2 files changed, 10 insertions(+), 5 deletions(-) rename ietf/doc/{tests_unprepped.py => tests_notprepped.py} (92%) diff --git a/ietf/doc/tests_unprepped.py b/ietf/doc/tests_notprepped.py similarity index 92% rename from ietf/doc/tests_unprepped.py rename to ietf/doc/tests_notprepped.py index f88af8e81a..f417aa7931 100644 --- a/ietf/doc/tests_unprepped.py +++ b/ietf/doc/tests_notprepped.py @@ -12,7 +12,7 @@ from ietf.utils.test_utils import TestCase -class UnpreppedRfcXmlTests(TestCase): +class NotpreppedRfcXmlTests(TestCase): def test_editor_source_button_visibility(self): pre_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC - 1) first_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC) @@ -72,13 +72,17 @@ def test_rfcxml_notprepped(self): r = self.client.get(url) self.assertEqual(r.status_code, 404) - # 200 with correct content-type and body when object is fully stored + # 200 with correct content-type, attachment disposition, and body when object is fully stored xml_content = b"test" store_bytes("rfc", stored_name, xml_content, allow_overwrite=True) r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertEqual(r["Content-Type"], "application/xml") - self.assertEqual(r.content, xml_content) + self.assertEqual( + r["Content-Disposition"], + f'attachment; filename="rfc{number}.notprepped.xml"', + ) + self.assertEqual(b"".join(r.streaming_content), xml_content) def test_rfcxml_notprepped_wrapper(self): number = settings.FIRST_V3_RFC diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index a23185333e..5b57a62074 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -43,9 +43,10 @@ from celery.result import AsyncResult from django.core.cache import caches +from django.core.files.base import ContentFile from django.core.exceptions import PermissionDenied from django.db.models import Max -from django.http import HttpResponse, Http404, HttpResponseBadRequest, JsonResponse +from django.http import FileResponse, HttpResponse, Http404, HttpResponseBadRequest, JsonResponse from django.shortcuts import render, get_object_or_404, redirect from django.template.loader import render_to_string from django.urls import reverse as urlreverse @@ -2372,7 +2373,7 @@ def rfcxml_notprepped(request, number): bytes = retrieve_bytes("rfc", name) except FileNotFoundError: raise Http404 - return HttpResponse(bytes, content_type="application/xml") + return FileResponse(ContentFile(bytes, name=f"rfc{number}.notprepped.xml"), as_attachment=True) def rfcxml_notprepped_wrapper(request, number): From 9cecc36bc7e42ecc5cd196d96f4bd0eaf03b5e69 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Apr 2026 00:25:02 -0300 Subject: [PATCH 03/17] feat: rebuild_searchindex task (#10723) * refactor: DRY * chore: typesense docker container (commented out) * feat: batched RFC search index import * feat: rebuild_searchindex task * feat: logging / error reporting * refactor: _task suffix for task name * test: tests for searchindex utils + tasks * fix: only create collection if dropped * fix: typing / lint --- docker-compose.yml | 12 ++ ietf/doc/tasks.py | 11 ++ ietf/doc/tests_tasks.py | 43 ++++++ ietf/utils/searchindex.py | 239 ++++++++++++++++++++++++++++++-- ietf/utils/tests_searchindex.py | 152 +++++++++++++++----- 5 files changed, 410 insertions(+), 47 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4c3f2f6b8e..073d04b896 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -132,6 +132,18 @@ services: volumes: - blobdb-data:/var/lib/postgresql/data +# typesense: +# image: typesense/typesense:30.1 +# restart: on-failure +# ports: +# - "8108:8108" +# volumes: +# - ./typesense-data:/data +# command: +# - '--data-dir=/data' +# - '--api-key=typesense-api-key' +# - '--enable-cors' + # Celery Beat is a periodic task runner. It is not normally needed for development, # but can be enabled by uncommenting the following. # diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index 19edb39014..273242e35f 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -209,3 +209,14 @@ def update_rfc_searchindex_task(self, rfc_number: int): countdown=searchindex_settings["TASK_RETRY_DELAY"], max_retries=searchindex_settings["TASK_MAX_RETRIES"], ) + + +@shared_task +def rebuild_searchindex_task(*, batchsize=40, drop_collection=False): + if drop_collection: + searchindex.delete_collection() + searchindex.create_collection() + searchindex.update_or_create_rfc_entries( + Document.objects.filter(type_id="rfc").order_by("-rfc_number"), + batchsize=batchsize, + ) diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 728d21f131..2e2d65463f 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -24,6 +24,7 @@ generate_idnits2_rfc_status_task, investigate_fragment_task, notify_expirations_task, + rebuild_searchindex_task, update_rfc_searchindex_task, ) @@ -144,6 +145,48 @@ def test_update_rfc_searchindex_task( with self.assertRaises(Retry): update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + @mock.patch("ietf.doc.tasks.searchindex.update_or_create_rfc_entries") + @mock.patch("ietf.doc.tasks.searchindex.create_collection") + @mock.patch("ietf.doc.tasks.searchindex.delete_collection") + def test_rebuild_searchindex_task(self, mock_delete, mock_create, mock_update): + rfcs = WgRfcFactory.create_batch(10) + rebuild_searchindex_task() + self.assertFalse(mock_delete.called) + self.assertFalse(mock_create.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + + mock_delete.reset_mock() + mock_create.reset_mock() + mock_update.reset_mock() + rebuild_searchindex_task(drop_collection=True) + self.assertTrue(mock_delete.called) + self.assertTrue(mock_create.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + + mock_delete.reset_mock() + mock_create.reset_mock() + mock_update.reset_mock() + rebuild_searchindex_task(drop_collection=True, batchsize=3) + self.assertTrue(mock_delete.called) + self.assertTrue(mock_create.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + self.assertEqual(mock_update.call_args.kwargs["batchsize"], 3) + class Idnits2SupportTests(TestCase): settings_temp_path_overrides = TestCase.settings_temp_path_overrides + [ diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py index e4427b88b5..a47e6d2f12 100644 --- a/ietf/utils/searchindex.py +++ b/ietf/utils/searchindex.py @@ -2,12 +2,15 @@ """Search indexing utilities""" import re +from itertools import batched from math import floor +from typing import Iterable import httpx # just for exceptions import typesense import typesense.exceptions from django.conf import settings +from typesense.types.document import DocumentSchema from ietf.doc.models import Document, StoredObject from ietf.doc.storage_utils import retrieve_str @@ -42,6 +45,24 @@ def enabled(): return _settings["TYPESENSE_API_URL"] != "" +def get_typesense_client() -> typesense.Client: + _settings = get_settings() + client = typesense.Client( + { + "api_key": _settings["TYPESENSE_API_KEY"], + "nodes": [_settings["TYPESENSE_API_URL"]], + } + ) + return client + + +def get_collection_name() -> str: + _settings = get_settings() + collection_name = _settings["TYPESENSE_COLLECTION_NAME"] + assert isinstance(collection_name, str) + return collection_name + + def _sanitize_text(content): """Sanitize content or abstract text for search""" # REs (with approximate names) @@ -62,7 +83,7 @@ def _sanitize_text(content): return content.strip() -def update_or_create_rfc_entry(rfc: Document): +def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: assert rfc.type_id == "rfc" assert rfc.rfc_number is not None @@ -75,8 +96,8 @@ def update_or_create_rfc_entry(rfc: Document): f"Indexing as {subseries[0].name}" ) subseries = subseries[0] if len(subseries) > 0 else None - obsoleted_by = rfc.relations_that("obs") - updated_by = rfc.relations_that("updates") + obsoleted_by = rfc.related_that("obs") + updated_by = rfc.related_that("updates") stored_txt = ( StoredObject.objects.exclude_deleted() @@ -91,8 +112,8 @@ def update_or_create_rfc_entry(rfc: Document): except Exception as err: log(f"Unable to retrieve {stored_txt} from storage: {err}") - ts_id = f"doc-{rfc.pk}" ts_document = { + "id": f"doc-{rfc.pk}", "rfcNumber": rfc.rfc_number, "rfc": str(rfc.rfc_number), "filename": rfc.name, @@ -143,13 +164,205 @@ def update_or_create_rfc_entry(rfc: Document): ts_document["adName"] = rfc.ad.name if content != "": ts_document["content"] = _sanitize_text(content) - _settings = get_settings() - client = typesense.Client( + return ts_document + + +def update_or_create_rfc_entry(rfc: Document): + """Update/create index entries for one RFC""" + ts_document = typesense_doc_from_rfc(rfc) + client = get_typesense_client() + client.collections[get_collection_name()].documents.upsert(ts_document) + + +def update_or_create_rfc_entries( + rfcs: Iterable[Document], batchsize: int | None = None +): + """Update/create index entries for RFCs in bulk + + If batchsize is set, computes index data in batches of batchsize and adds to the + index. Will make a total of (len(rfcs) // batchsize) + 1 API calls. + + N.b. that typesense has a server-side batch size that defaults to 40, which should + "almost never be changed from the default." This does not change that. Further, + the python client library's import_ method has a batch_size parameter that does + client-side batching. We don't use that, either. + """ + success_count = 0 + fail_count = 0 + client = get_typesense_client() + batches = [rfcs] if batchsize is None else batched(rfcs, batchsize) + for batch in batches: + tdoc_batch = [typesense_doc_from_rfc(rfc) for rfc in batch] + results = client.collections[get_collection_name()].documents.import_( + tdoc_batch, {"action": "upsert"} + ) + for tdoc, result in zip(tdoc_batch, results): + if result["success"]: + success_count += 1 + else: + fail_count += 1 + log(f"Failed to index RFC {tdoc['rfcNumber']}: {result['error']}") + log(f"Added {success_count} RFCs to the index, failed to add {fail_count}") + + +DOCS_SCHEMA = { + "enable_nested_fields": True, + "default_sorting_field": "ranking", + "fields": [ + # RFC number in integer form, for sorting asc/desc in search results + # Omit field for drafts { - "api_key": _settings["TYPESENSE_API_KEY"], - "nodes": [_settings["TYPESENSE_API_URL"]], - } - ) - client.collections[_settings["TYPESENSE_COLLECTION_NAME"]].documents.upsert( - {"id": ts_id} | ts_document - ) + "name": "rfcNumber", + "type": "int32", + "facet": False, + "optional": True, + "sort": True, + }, + # RFC number in string form, for direct matching with ranking + # Omit field for drafts + {"name": "rfc", "type": "string", "facet": False, "optional": True}, + # For drafts that correspond to an RFC, insert the RFC number + # Omit field for rfcs or if not relevant + {"name": "ref", "type": "string", "facet": False, "optional": True}, + # Filename of the document (without the extension, e.g. "rfc1234" + # or "draft-ietf-abc-def-02") + {"name": "filename", "type": "string", "facet": False, "infix": True}, + # Title of the draft / rfc + {"name": "title", "type": "string", "facet": False}, + # Abstract of the draft / rfc + {"name": "abstract", "type": "string", "facet": False}, + # A list of search keywords if relevant, set to empty array otherwise + {"name": "keywords", "type": "string[]", "facet": True}, + # Type of the document + # Accepted values: "draft" or "rfc" + {"name": "type", "type": "string", "facet": True}, + # State(s) of the document (e.g. "Published", "Adopted by a WG", etc.) + # Use the full name, not the slug + {"name": "state", "type": "string[]", "facet": True, "optional": True}, + # Status (Standard Level Name) + # Object with properties "slug" and "name" + # e.g.: { slug: "std", "name": "Internet Standard" } + {"name": "status", "type": "object", "facet": True, "optional": True}, + # The subseries it is part of. (e.g. "BCP") + # Omit otherwise. + { + "name": "subseries.acronym", + "type": "string", + "facet": True, + "optional": True, + }, + # The subseries number it is part of. (e.g. 123) + # Omit otherwise. + { + "name": "subseries.number", + "type": "int32", + "facet": True, + "sort": True, + "optional": True, + }, + # The total of RFCs in the subseries + # Omit if not part of a subseries + { + "name": "subseries.total", + "type": "int32", + "facet": False, + "sort": False, + "optional": True, + }, + # Date of the document, in unix epoch seconds (can be negative for < 1970) + {"name": "date", "type": "int64", "facet": False}, + # Expiration date of the document, in unix epoch seconds (can be negative + # for < 1970). Omit field for RFCs + {"name": "expires", "type": "int64", "facet": False, "optional": True}, + # Publication date of the RFC, in unix epoch seconds (can be negative + # for < 1970). Omit field for drafts + { + "name": "publicationDate", + "type": "int64", + "facet": True, + "optional": True, + }, + # Working Group + # Object with properties "acronym", "name" and "full" + # e.g.: + # { + # "acronym": "ntp", + # "name": "Network Time Protocols", + # "full": "ntp - Network Time Protocols", + # } + {"name": "group", "type": "object", "facet": True, "optional": True}, + # Area + # Object with properties "acronym", "name" and "full" + # e.g.: + # { + # "acronym": "mpls", + # "name": "Multiprotocol Label Switching", + # "full": "mpls - Multiprotocol Label Switching", + # } + {"name": "area", "type": "object", "facet": True, "optional": True}, + # Stream + # Object with properties "slug" and "name" + # e.g.: { slug: "ietf", "name": "IETF" } + {"name": "stream", "type": "object", "facet": True, "optional": True}, + # List of authors + # Array of objects with properties "name" and "affiliation" + # e.g.: + # [ + # {"name": "John Doe", "affiliation": "ACME Inc."}, + # {"name": "Ada Lovelace", "affiliation": "Babbage Corps."}, + # ] + {"name": "authors", "type": "object[]", "facet": True, "optional": True}, + # Area Director Name (e.g. "Leonardo DaVinci") + {"name": "adName", "type": "string", "facet": True, "optional": True}, + # Whether the document should be hidden by default in search results or not. + {"name": "flags.hiddenDefault", "type": "bool", "facet": True}, + # Whether the document is obsoleted by another document or not. + {"name": "flags.obsoleted", "type": "bool", "facet": True}, + # Whether the document is updated by another document or not. + {"name": "flags.updated", "type": "bool", "facet": True}, + # List of documents that obsolete this document. + # Array of strings. Use RFC number for RFCs. (e.g. ["123", "456"]) + # Omit if none. Must be provided if "flags.obsoleted" is set to True. + { + "name": "obsoletedBy", + "type": "string[]", + "facet": False, + "optional": True, + }, + # List of documents that update this document. + # Array of strings. Use RFC number for RFCs. (e.g. ["123", "456"]) + # Omit if none. Must be provided if "flags.updated" is set to True. + {"name": "updatedBy", "type": "string[]", "facet": False, "optional": True}, + # Sanitized content of the document. + # Make sure to remove newlines, double whitespaces, symbols and tags. + { + "name": "content", + "type": "string", + "facet": False, + "optional": True, + "store": False, + }, + # Ranking value to use when no explicit sorting is used during search + # Set to the RFC number for RFCs and the revision number for drafts + # This ensures newer RFCs get listed first in the default search results + # (without a query) + {"name": "ranking", "type": "int32", "facet": False}, + ], +} + + +def create_collection(): + collection_name = get_collection_name() + log(f"Creating '{collection_name}' collection") + client = get_typesense_client() + client.collections.create({"name": get_collection_name()} | DOCS_SCHEMA) + + +def delete_collection(): + collection_name = get_collection_name() + log(f"Deleting '{collection_name}' collection") + client = get_typesense_client() + try: + client.collections[collection_name].delete() + except typesense.exceptions.ObjectNotFound: + pass diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py index 8740716c85..0bff96ec7d 100644 --- a/ietf/utils/tests_searchindex.py +++ b/ietf/utils/tests_searchindex.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2026, All Rights Reserved from unittest import mock +import typesense.exceptions from django.conf import settings from django.test.utils import override_settings @@ -51,42 +52,29 @@ def test_sanitize_text(self): "TYPESENSE_COLLECTION_NAME": "frogs", } ) - @mock.patch("ietf.utils.searchindex.typesense.Client") - def test_update_or_create_rfc_entry(self, mock_ts_client_constructor): + def test_typesense_doc_from_rfc(self): not_rfc = WgDraftFactory() assert isinstance(not_rfc, Document) with self.assertRaises(AssertionError): - searchindex.update_or_create_rfc_entry(not_rfc) - self.assertFalse(mock_ts_client_constructor.called) + searchindex.typesense_doc_from_rfc(not_rfc) invalid_rfc = WgRfcFactory(name="rfc1000000", rfc_number=None) assert isinstance(invalid_rfc, Document) with self.assertRaises(AssertionError): - searchindex.update_or_create_rfc_entry(invalid_rfc) - self.assertFalse(mock_ts_client_constructor.called) + searchindex.typesense_doc_from_rfc(invalid_rfc) rfc = PublishedRfcDocEventFactory().doc assert isinstance(rfc, Document) - searchindex.update_or_create_rfc_entry(rfc) - self.assertTrue(mock_ts_client_constructor.called) - # walk the tree down to the method we expected to be called... - mock_upsert = mock_ts_client_constructor.return_value.collections[ - "frogs" - ].documents.upsert # matches value in override_settings above - self.assertTrue(mock_upsert.called) - upserted_dict = mock_upsert.call_args[0][0] + result = searchindex.typesense_doc_from_rfc(rfc) # Check a few values, not exhaustive - self.assertEqual(upserted_dict["id"], f"doc-{rfc.pk}") - self.assertEqual(upserted_dict["rfcNumber"], rfc.rfc_number) - self.assertEqual( - upserted_dict["abstract"], searchindex._sanitize_text(rfc.abstract) - ) - self.assertNotIn("adName", upserted_dict) - self.assertNotIn("content", upserted_dict) # no blob - self.assertNotIn("subseries", upserted_dict) + self.assertEqual(result["id"], f"doc-{rfc.pk}") + self.assertEqual(result["rfcNumber"], rfc.rfc_number) + self.assertEqual(result["abstract"], searchindex._sanitize_text(rfc.abstract)) + self.assertNotIn("adName", result) + self.assertNotIn("content", result) # no blob + self.assertNotIn("subseries", result) # repeat, this time with contents, an AD, and subseries docs - mock_upsert.reset_mock() store_str( kind="rfc", name=f"txt/{rfc.name}.txt", @@ -99,17 +87,15 @@ def test_update_or_create_rfc_entry(self, mock_ts_client_constructor): # (the typesense schema does not support this for real at the moment) BcpFactory(contains=[rfc], name="bcp1234") StdFactory(contains=[rfc], name="std1234") - searchindex.update_or_create_rfc_entry(rfc) - self.assertTrue(mock_upsert.called) - upserted_dict = mock_upsert.call_args[0][0] + result = searchindex.typesense_doc_from_rfc(rfc) # Check a few values, not exhaustive self.assertEqual( - upserted_dict["content"], + result["content"], searchindex._sanitize_text("The contents of this RFC"), ) - self.assertEqual(upserted_dict["adName"], "Alfred D. Rector") - self.assertIn("subseries", upserted_dict) - ss_dict = upserted_dict["subseries"] + self.assertEqual(result["adName"], "Alfred D. Rector") + self.assertIn("subseries", result) + ss_dict = result["subseries"] # We should get one of the two subseries docs, but neither is more correct # than the other... self.assertTrue( @@ -119,10 +105,108 @@ def test_update_or_create_rfc_entry(self, mock_ts_client_constructor): ) ) - # Finally, delete the contents blob and make sure things don't blow up - mock_upsert.reset_mock() + # Finally, delete the contents blob and make sure things don't blow up Blob.objects.get(bucket="rfc", name=f"txt/{rfc.name}.txt").delete() + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertNotIn("content", result) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense_doc_from_rfc") + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_update_or_create_rfc_entry( + self, mock_ts_client_constructor, mock_tdoc_from_rfc + ): + fake_tdoc = object() + mock_tdoc_from_rfc.return_value = fake_tdoc + rfc = WgRfcFactory() + assert isinstance(rfc, Document) searchindex.update_or_create_rfc_entry(rfc) + self.assertTrue(mock_ts_client_constructor.called) + # walk the tree down to the method we expected to be called... + mock_upsert = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.upsert self.assertTrue(mock_upsert.called) - upserted_dict = mock_upsert.call_args[0][0] - self.assertNotIn("content", upserted_dict) + self.assertEqual(mock_upsert.call_args, mock.call(fake_tdoc)) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense_doc_from_rfc") + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_update_or_create_rfc_entries( + self, mock_ts_client_constructor, mock_tdoc_from_rfc + ): + fake_tdoc = object() + mock_tdoc_from_rfc.return_value = fake_tdoc + rfc = WgRfcFactory() + assert isinstance(rfc, Document) + searchindex.update_or_create_rfc_entries([rfc] * 50) # list of docs... + self.assertEqual(mock_ts_client_constructor.call_count, 1) + # walk the tree down to the method we expected to be called... + mock_import_ = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.import_ + self.assertEqual(mock_import_.call_count, 1) + self.assertEqual( + mock_import_.call_args, mock.call([fake_tdoc] * 50, {"action": "upsert"}) + ) + + mock_import_.reset_mock() + searchindex.update_or_create_rfc_entries([rfc] * 50, batchsize=20) + self.assertEqual(mock_ts_client_constructor.call_count, 2) # one more + # walk the tree down to the method we expected to be called... + mock_import_ = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.import_ + self.assertEqual(mock_import_.call_count, 3) + self.assertEqual( + mock_import_.call_args_list, + [ + mock.call([fake_tdoc] * 20, {"action": "upsert"}), + mock.call([fake_tdoc] * 20, {"action": "upsert"}), + mock.call([fake_tdoc] * 10, {"action": "upsert"}), + ], + ) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_create_collection(self, mock_ts_client_constructor): + searchindex.create_collection() + self.assertEqual(mock_ts_client_constructor.call_count, 1) + mock_collections = mock_ts_client_constructor.return_value.collections + self.assertTrue(mock_collections.create.called) + self.assertEqual(mock_collections.create.call_args[0][0]["name"], "frogs") + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_delete_collection(self, mock_ts_client_constructor): + searchindex.delete_collection() + self.assertEqual(mock_ts_client_constructor.call_count, 1) + mock_collections = mock_ts_client_constructor.return_value.collections + self.assertTrue(mock_collections["frogs"].delete.called) + + mock_collections["frogs"].side_effect = typesense.exceptions.ObjectNotFound + searchindex.delete_collection() # should ignore the exception From c4cb8b91fc9434a3bb3419acfac2dd3b30cb4a6c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Apr 2026 07:33:51 -0300 Subject: [PATCH 04/17] fix: add pages to typesense schema (#10726) --- ietf/utils/searchindex.py | 4 ++++ ietf/utils/tests_searchindex.py | 1 + 2 files changed, 5 insertions(+) diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py index a47e6d2f12..87951abb60 100644 --- a/ietf/utils/searchindex.py +++ b/ietf/utils/searchindex.py @@ -86,6 +86,7 @@ def _sanitize_text(content): def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: assert rfc.type_id == "rfc" assert rfc.rfc_number is not None + assert rfc.pages is not None keywords: list[str] = rfc.keywords # help type checking @@ -119,6 +120,7 @@ def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: "filename": rfc.name, "title": rfc.title, "abstract": _sanitize_text(rfc.abstract), + "pages": rfc.pages, "keywords": keywords, "type": "rfc", "state": [state.name for state in rfc.states.all()], @@ -231,6 +233,8 @@ def update_or_create_rfc_entries( {"name": "title", "type": "string", "facet": False}, # Abstract of the draft / rfc {"name": "abstract", "type": "string", "facet": False}, + # Number of pages + {"name": "pages", "type": "int32", "facet": False}, # A list of search keywords if relevant, set to empty array otherwise {"name": "keywords", "type": "string[]", "facet": True}, # Type of the document diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py index 0bff96ec7d..e9fbf52020 100644 --- a/ietf/utils/tests_searchindex.py +++ b/ietf/utils/tests_searchindex.py @@ -70,6 +70,7 @@ def test_typesense_doc_from_rfc(self): self.assertEqual(result["id"], f"doc-{rfc.pk}") self.assertEqual(result["rfcNumber"], rfc.rfc_number) self.assertEqual(result["abstract"], searchindex._sanitize_text(rfc.abstract)) + self.assertEqual(result["pages"], rfc.pages) self.assertNotIn("adName", result) self.assertNotIn("content", result) # no blob self.assertNotIn("subseries", result) From 629ffb13480201e25fc5d941cfcea9de123562f9 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Apr 2026 15:04:23 -0300 Subject: [PATCH 05/17] fix: decode non-utf-8 blob content (#10729) * refactor: decode_document_content() utility method * fix: fall back to latin-1 in retrieve_str() * refactor: match structure with retrieve_bytes() * refactor: separate tests_text.py module * test: test_decode_document_content + ruff * fix: revert misguided refactor * test: assert to guarantee test is valid --- ietf/doc/models.py | 15 ++------- ietf/doc/storage_utils.py | 47 +++++++++++++------------- ietf/utils/tests.py | 19 ----------- ietf/utils/tests_text.py | 71 +++++++++++++++++++++++++++++++++++++++ ietf/utils/text.py | 18 ++++++++++ 5 files changed, 114 insertions(+), 56 deletions(-) create mode 100644 ietf/utils/tests_text.py diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 972f0a34e8..cc79b73831 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -52,6 +52,7 @@ from ietf.person.utils import get_active_balloters from ietf.utils import log from ietf.utils.decorators import memoize +from ietf.utils.text import decode_document_content from ietf.utils.validators import validate_no_control_chars from ietf.utils.mail import formataddr from ietf.utils.models import ForeignKey @@ -640,19 +641,7 @@ def text(self, size = -1): except IOError as e: log.log(f"Error reading text for {path}: {e}") return None - text = None - try: - text = raw.decode('utf-8') - except UnicodeDecodeError: - for back in range(1,4): - try: - text = raw[:-back].decode('utf-8') - break - except UnicodeDecodeError: - pass - if text is None: - text = raw.decode('latin-1') - return text + return decode_document_content(raw) def text_or_error(self): return self.text() or "Error; cannot read '%s'"%self.get_base_name() diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index ffdd4599be..9c18bb8a8a 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -10,6 +10,7 @@ from django.core.files.storage import storages, Storage from ietf.utils.log import log +from ietf.utils.text import decode_document_content class StorageUtilsError(Exception): @@ -164,32 +165,30 @@ def store_str( def retrieve_bytes(kind: str, name: str) -> bytes: from ietf.doc.storage import maybe_log_timing - content = b"" - if settings.ENABLE_BLOBSTORAGE: - try: - store = _get_storage(kind) - with store.open(name) as f: - with maybe_log_timing( - hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing, - "read", - bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "", - name=name, - ): - content = f.read() - except Exception as err: - log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}") - raise + if not settings.ENABLE_BLOBSTORAGE: + return b"" + try: + store = _get_storage(kind) + with store.open(name) as f: + with maybe_log_timing( + hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing, + "read", + bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "", + name=name, + ): + content = f.read() + except Exception as err: + log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}") + raise return content def retrieve_str(kind: str, name: str) -> str: - content = "" - if settings.ENABLE_BLOBSTORAGE: - try: - content_bytes = retrieve_bytes(kind, name) - # TODO-BLOBSTORE: try to decode all the different ways doc.text() does - content = content_bytes.decode("utf-8") - except Exception as err: - log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}") - raise + if not settings.ENABLE_BLOBSTORAGE: + return "" + try: + content = decode_document_content(retrieve_bytes(kind, name)) + except Exception as err: + log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}") + raise return content diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 3288309095..99c33f34b3 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -60,7 +60,6 @@ set_url_coverage, ) from ietf.utils.test_utils import TestCase, unicontent -from ietf.utils.text import parse_unicode from ietf.utils.timezone import timezone_not_near_midnight from ietf.utils.xmldraft import XMLDraft, InvalidMetadataError, capture_xml2rfc_output @@ -864,24 +863,6 @@ def test_assertion(self): assertion('False') settings.SERVER_MODE = 'test' -class TestRFC2047Strings(TestCase): - def test_parse_unicode(self): - names = ( - ('=?utf-8?b?4Yuz4YuK4Ym1IOGJoOGJgOGIiA==?=', 'ዳዊት በቀለ'), - ('=?utf-8?b?5Li9IOmDnA==?=', '丽 郜'), - ('=?utf-8?b?4KSV4KSu4KWN4KSs4KWL4KScIOCkoeCkvuCksA==?=', 'कम्बोज डार'), - ('=?utf-8?b?zpfPgc6szrrOu861zrnOsSDOm865z4zOvc+Ezrc=?=', 'Ηράκλεια Λιόντη'), - ('=?utf-8?b?15nXqdeo15DXnCDXqNeV15bXoNek15zXkw==?=', 'ישראל רוזנפלד'), - ('=?utf-8?b?5Li95Y2OIOeahw==?=', '丽华 皇'), - ('=?utf-8?b?77ul77qu766V77qzIO+tlu+7ru+vvu+6ju+7pw==?=', 'ﻥﺮﮕﺳ ﭖﻮﯾﺎﻧ'), - ('=?utf-8?b?77uh77uu77qz77uu76++IO+6su+7tO+7p++6jSDvurDvu6Pvuo7vu6jvr74=?=', 'ﻡﻮﺳﻮﯾ ﺲﻴﻧﺍ ﺰﻣﺎﻨﯾ'), - ('=?utf-8?b?ScOxaWdvIFNhbsOnIEliw6HDsWV6IGRlIGxhIFBlw7Fh?=', 'Iñigo Sanç Ibáñez de la Peña'), - ('Mart van Oostendorp', 'Mart van Oostendorp'), - ('', ''), - ) - for encoded_str, unicode in names: - self.assertEqual(unicode, parse_unicode(encoded_str)) - class TestAndroidSiteManifest(TestCase): def test_manifest(self): r = self.client.get(urlreverse('site.webmanifest')) diff --git a/ietf/utils/tests_text.py b/ietf/utils/tests_text.py new file mode 100644 index 0000000000..51aa2eff13 --- /dev/null +++ b/ietf/utils/tests_text.py @@ -0,0 +1,71 @@ +# Copyright The IETF Trust 2021-2026, All Rights Reserved +from ietf.utils.test_utils import TestCase +from ietf.utils.text import parse_unicode, decode_document_content + + +class TestDecoders(TestCase): + def test_parse_unicode(self): + names = ( + ("=?utf-8?b?4Yuz4YuK4Ym1IOGJoOGJgOGIiA==?=", "ዳዊት በቀለ"), + ("=?utf-8?b?5Li9IOmDnA==?=", "丽 郜"), + ("=?utf-8?b?4KSV4KSu4KWN4KSs4KWL4KScIOCkoeCkvuCksA==?=", "कम्बोज डार"), + ("=?utf-8?b?zpfPgc6szrrOu861zrnOsSDOm865z4zOvc+Ezrc=?=", "Ηράκλεια Λιόντη"), + ("=?utf-8?b?15nXqdeo15DXnCDXqNeV15bXoNek15zXkw==?=", "ישראל רוזנפלד"), + ("=?utf-8?b?5Li95Y2OIOeahw==?=", "丽华 皇"), + ("=?utf-8?b?77ul77qu766V77qzIO+tlu+7ru+vvu+6ju+7pw==?=", "ﻥﺮﮕﺳ ﭖﻮﯾﺎﻧ"), + ( + "=?utf-8?b?77uh77uu77qz77uu76++IO+6su+7tO+7p++6jSDvurDvu6Pvuo7vu6jvr74=?=", + "ﻡﻮﺳﻮﯾ ﺲﻴﻧﺍ ﺰﻣﺎﻨﯾ", + ), + ( + "=?utf-8?b?ScOxaWdvIFNhbsOnIEliw6HDsWV6IGRlIGxhIFBlw7Fh?=", + "Iñigo Sanç Ibáñez de la Peña", + ), + ("Mart van Oostendorp", "Mart van Oostendorp"), + ("", ""), + ) + for encoded_str, unicode in names: + self.assertEqual(unicode, parse_unicode(encoded_str)) + + def test_decode_document_content(self): + utf8_bytes = "𒀭𒊩𒌆𒄈𒋢".encode("utf-8") # ends with 4-byte character + latin1_bytes = "àéîøü".encode("latin-1") + other_bytes = "àéîøü".encode("macintosh") # different from its latin-1 encoding + assert other_bytes.decode("macintosh") != other_bytes.decode("latin-1"),\ + "test broken: other_bytes must decode differently as latin-1" + + # simplest case + self.assertEqual( + decode_document_content(utf8_bytes), + utf8_bytes.decode(), + ) + # losing 1-4 bytes from the end leave the last character incomplete; the + # decoder should decode all but that last character + self.assertEqual( + decode_document_content(utf8_bytes[:-1]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-2]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-3]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-4]), + utf8_bytes.decode()[:-1], + ) + + # latin-1 is also simple + self.assertEqual( + decode_document_content(latin1_bytes), + latin1_bytes.decode("latin-1"), + ) + + # other character sets are just treated as latin1 (bug? feature? you decide) + self.assertEqual( + decode_document_content(other_bytes), + other_bytes.decode("latin-1"), + ) diff --git a/ietf/utils/text.py b/ietf/utils/text.py index 590ec3fd30..2763056e1a 100644 --- a/ietf/utils/text.py +++ b/ietf/utils/text.py @@ -263,3 +263,21 @@ def parse_unicode(text): else: text = decoded_string return text + + +def decode_document_content(content: bytes) -> str: + """Decode document contents as utf-8 or latin1 + + Method was developed in DocumentInfo.text() where it gave acceptable results + for existing documents / RFCs. + """ + try: + return content.decode("utf-8") + except UnicodeDecodeError: + pass + for back in range(1, 4): + try: + return content[:-back].decode("utf-8") + except UnicodeDecodeError: + pass + return content.decode("latin-1") # everything is legal in latin-1 From 63a69945ab11b1c3b3ec490fb260073c90eed0bc Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 17 Apr 2026 16:24:18 -0500 Subject: [PATCH 06/17] test: Squash some transient test error vectors (#10730) * test: enforce queryset order assumed by test * test: match html escaping in test * test: search more specifically for tokens to avoid mis-reading them when they occur in faker data --- ietf/group/tests_review.py | 30 +++++++++++++------------- ietf/meeting/tests_session_requests.py | 2 +- ietf/meeting/tests_views.py | 7 +++--- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/ietf/group/tests_review.py b/ietf/group/tests_review.py index 89c755bb26..bb9b79a416 100644 --- a/ietf/group/tests_review.py +++ b/ietf/group/tests_review.py @@ -888,10 +888,10 @@ def test_requests_history_filter_page(self): self.assertEqual(r.status_code, 200) self.assertContains(r, review_req.doc.name) self.assertContains(r, review_req2.doc.name) - self.assertContains(r, 'Assigned') - self.assertContains(r, 'Accepted') - self.assertContains(r, 'Completed') - self.assertContains(r, 'Ready') + self.assertContains(r, 'data-text="Assigned"') + self.assertContains(r, 'data-text="Accepted"') + self.assertContains(r, 'data-text="Completed"') + self.assertContains(r, 'data-text="Ready"') self.assertContains(r, escape(assignment.reviewer.person.name)) self.assertContains(r, escape(assignment2.reviewer.person.name)) @@ -907,10 +907,10 @@ def test_requests_history_filter_page(self): self.assertEqual(r.status_code, 200) self.assertContains(r, review_req.doc.name) self.assertNotContains(r, review_req2.doc.name) - self.assertContains(r, 'Assigned') - self.assertNotContains(r, 'Accepted') - self.assertNotContains(r, 'Completed') - self.assertNotContains(r, 'Ready') + self.assertContains(r, 'data-text="Assigned"') + self.assertNotContains(r, 'data-text="Accepted"') + self.assertNotContains(r, 'data-text="Completed"') + self.assertNotContains(r, 'data-text="Ready"') self.assertContains(r, escape(assignment.reviewer.person.name)) self.assertNotContains(r, escape(assignment2.reviewer.person.name)) @@ -926,10 +926,10 @@ def test_requests_history_filter_page(self): self.assertEqual(r.status_code, 200) self.assertNotContains(r, review_req.doc.name) self.assertContains(r, review_req2.doc.name) - self.assertNotContains(r, 'Assigned') - self.assertContains(r, 'Accepted') - self.assertContains(r, 'Completed') - self.assertContains(r, 'Ready') + self.assertNotContains(r, 'data-text="Assigned"') + self.assertContains(r, 'data-text="Accepted"') + self.assertContains(r, 'data-text="Completed"') + self.assertContains(r, 'data-text="Ready"') self.assertNotContains(r, escape(assignment.reviewer.person.name)) self.assertContains(r, escape(assignment2.reviewer.person.name)) @@ -940,9 +940,9 @@ def test_requests_history_filter_page(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertNotContains(r, review_req.doc.name) - self.assertNotContains(r, 'Assigned') - self.assertNotContains(r, 'Accepted') - self.assertNotContains(r, 'Completed') + self.assertNotContains(r, 'data-text="Assigned"') + self.assertNotContains(r, 'data-text="Accepted"') + self.assertNotContains(r, 'data-text="Completed"') def test_requests_history_invalid_filter_parameters(self): # First assignment as assigned diff --git a/ietf/meeting/tests_session_requests.py b/ietf/meeting/tests_session_requests.py index 0cb092d2f8..42dbee5f23 100644 --- a/ietf/meeting/tests_session_requests.py +++ b/ietf/meeting/tests_session_requests.py @@ -236,7 +236,7 @@ def test_edit(self): self.assertRedirects(r, redirect_url) # Check whether updates were stored in the database - sessions = Session.objects.filter(meeting=meeting, group=mars) + sessions = Session.objects.filter(meeting=meeting, group=mars).order_by("id") self.assertEqual(len(sessions), 2) session = sessions[0] self.assertFalse(session.constraints().filter(name='time_relation')) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 258ffe554c..17988e50be 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -33,6 +33,7 @@ from django.http import QueryDict, FileResponse from django.template import Context, Template from django.utils import timezone +from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.text import slugify @@ -9491,7 +9492,7 @@ def test_session_attendance(self): self.assertEqual(r.status_code, 200) self.assertContains(r, '3 attendees') for person in persons: - self.assertContains(r, person.plain_name()) + self.assertContains(r, escape(person.plain_name())) # Test for the "I was there" button. def _test_button(person, expected): @@ -9511,14 +9512,14 @@ def _test_button(person, expected): # attempt to POST anyway is ignored r = self.client.post(attendance_url) self.assertEqual(r.status_code, 200) - self.assertNotContains(r, persons[3].plain_name()) + self.assertNotContains(r, escape(persons[3].plain_name())) self.assertEqual(session.attended_set.count(), 3) # button is shown, and POST is accepted meeting.importantdate_set.update(name_id='revsub',date=date_today() + datetime.timedelta(days=20)) _test_button(persons[3], True) r = self.client.post(attendance_url) self.assertEqual(r.status_code, 200) - self.assertContains(r, persons[3].plain_name()) + self.assertContains(r, escape(persons[3].plain_name())) self.assertEqual(session.attended_set.count(), 4) # When the meeting is finalized, a bluesheet file is generated, From dc49dc8362812893cad560feecc55efcea1553dc Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 20 Apr 2026 14:29:41 -0300 Subject: [PATCH 07/17] chore: beat termination grace period -> 10 s (#10741) --- k8s/beat.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/beat.yaml b/k8s/beat.yaml index 9ab242681c..b4291c7e31 100644 --- a/k8s/beat.yaml +++ b/k8s/beat.yaml @@ -59,4 +59,4 @@ spec: name: files-cfgmap dnsPolicy: ClusterFirst restartPolicy: Always - terminationGracePeriodSeconds: 600 + terminationGracePeriodSeconds: 10 From 4d69329ef86054fa5bfb9da9acd0c966ab013d8f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 20 Apr 2026 23:59:36 -0300 Subject: [PATCH 08/17] chore: remove blobdb profiling logs (#10732) These are not useful any more, blobdb is fast --- ietf/doc/storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ietf/doc/storage.py b/ietf/doc/storage.py index 375620ccaf..ee1e76c4fa 100644 --- a/ietf/doc/storage.py +++ b/ietf/doc/storage.py @@ -114,7 +114,6 @@ def _get_write_parameters(self, name, content=None): class StoredObjectBlobdbStorage(BlobdbStorage): - ietf_log_blob_timing = True warn_if_missing = True # TODO-BLOBSTORE make this configurable (or remove it) def _save_stored_object(self, name, content) -> StoredObject: From e8e4dd65c325199e7f1a1b5f254c0359bd8b144d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 27 Apr 2026 17:15:14 -0300 Subject: [PATCH 09/17] feat: sync blobs with replica via admin (#10756) * refactor: factor out queue_for_replication() * feat: BlobdbStorage.force_replication() * feat: force_replication() in storage_utils.py * feat: replicate_stored_objects_for_document * feat: admin action to replicate Document blobs * feat: admin action to replicate StoredObject blobs * feat: admin action to replicate Blobs * fix: use get_blobdb() helper * fix: drop references to R2 in the admin --- ietf/blobdb/admin.py | 20 +++++++++++++++++++- ietf/blobdb/models.py | 27 ++++---------------------- ietf/blobdb/storage.py | 12 +++++++++++- ietf/blobdb/utils.py | 32 +++++++++++++++++++++++++++++++ ietf/doc/admin.py | 40 +++++++++++++++++++++++++++++++++++++-- ietf/doc/storage_utils.py | 9 +++++++++ ietf/doc/utils.py | 25 +++++++++++++++++++++++- 7 files changed, 137 insertions(+), 28 deletions(-) create mode 100644 ietf/blobdb/utils.py diff --git a/ietf/blobdb/admin.py b/ietf/blobdb/admin.py index 3e1a2a311f..44a30d1d7f 100644 --- a/ietf/blobdb/admin.py +++ b/ietf/blobdb/admin.py @@ -1,9 +1,12 @@ -# Copyright The IETF Trust 2025, All Rights Reserved +# Copyright The IETF Trust 2025-2026, All Rights Reserved from django.contrib import admin +from django.db.models import QuerySet from django.db.models.functions import Length from rangefilter.filters import DateRangeQuickSelectListFilterBuilder +from .apps import get_blobdb from .models import Blob, ResolvedMaterial +from .utils import queue_for_replication @admin.register(Blob) @@ -17,6 +20,7 @@ class BlobAdmin(admin.ModelAdmin): ] search_fields = ["name"] list_display_links = ["name"] + actions = ["replicate_blob"] def get_queryset(self, request): return ( @@ -30,6 +34,20 @@ def object_size(self, instance): """Get the size of the object""" return instance.object_size # annotation added in get_queryset() + @admin.action(description="Replicate blobs") + def replicate_blob(self, request, queryset: QuerySet[Blob]): + blob_count = 0 + for blob in queryset.all(): + if isinstance(blob, Blob): + queue_for_replication( + bucket=blob.bucket, name=blob.name, using=get_blobdb() + ) + blob_count += 1 + self.message_user( + request, + f"Queued replication of a total of {blob_count} Blob(s)", + ) + @admin.register(ResolvedMaterial) class ResolvedMaterialAdmin(admin.ModelAdmin): diff --git a/ietf/blobdb/models.py b/ietf/blobdb/models.py index 27325ada5d..6dbb615fa0 100644 --- a/ietf/blobdb/models.py +++ b/ietf/blobdb/models.py @@ -1,14 +1,11 @@ -# Copyright The IETF Trust 2025, All Rights Reserved -import json -from functools import partial +# Copyright The IETF Trust 2025-2026, All Rights Reserved from hashlib import sha384 from django.db import models, transaction from django.utils import timezone from .apps import get_blobdb -from .replication import replication_enabled -from .tasks import pybob_the_blob_replicator_task +from .utils import queue_for_replication class BlobQuerySet(models.QuerySet): @@ -81,24 +78,8 @@ def delete(self, **kwargs): self._emit_blob_change_event(using=db) return retval - def _emit_blob_change_event(self, using=None): - if not replication_enabled(self.bucket): - return - - # For now, fire a celery task we've arranged to guarantee in-order processing. - # Later becomes pushing an event onto a queue to a dedicated worker. - transaction.on_commit( - partial( - pybob_the_blob_replicator_task.delay, - json.dumps( - { - "name": self.name, - "bucket": self.bucket, - } - ) - ), - using=using, - ) + def _emit_blob_change_event(self, using: str | None=None): + queue_for_replication(self.bucket, self.name, using=using) class ResolvedMaterial(models.Model): diff --git a/ietf/blobdb/storage.py b/ietf/blobdb/storage.py index 4213ec801d..e304dabc5d 100644 --- a/ietf/blobdb/storage.py +++ b/ietf/blobdb/storage.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2025, All Rights Reserved +# Copyright The IETF Trust 2025-2026, All Rights Reserved from typing import Optional from django.core.exceptions import SuspiciousFileOperation @@ -10,6 +10,7 @@ from ietf.utils.storage import MetadataFile from .models import Blob +from .utils import queue_for_replication class BlobFile(MetadataFile): @@ -94,3 +95,12 @@ def get_available_name(self, name, max_length=None): f"asked to store the name '{name[:5]}...{name[-5:]} of length {len(name)}" ) return name # overwrite is permitted + + def force_replication(self, name: str): + """Force replication of a blob by name + + Be careful with this - replication includes replicating deletion of blobs, so + if you call it with a name that does not exist in blobdb, it will be removed + from R2 if it exists there! + """ + queue_for_replication(bucket=self.bucket_name, name=name) diff --git a/ietf/blobdb/utils.py b/ietf/blobdb/utils.py new file mode 100644 index 0000000000..93f8f2f521 --- /dev/null +++ b/ietf/blobdb/utils.py @@ -0,0 +1,32 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import json +from functools import partial + +from django.db import transaction + +from ietf.blobdb.replication import replication_enabled +from ietf.blobdb.tasks import pybob_the_blob_replicator_task + + +def queue_for_replication(bucket: str, name: str, using: str | None=None): + """Queue a blob for replication + + This is private to the blobdb app. Do not call it directly from other apps. + """ + if not replication_enabled(bucket): + return + + # For now, fire a celery task we've arranged to guarantee in-order processing. + # Later becomes pushing an event onto a queue to a dedicated worker. + transaction.on_commit( + partial( + pybob_the_blob_replicator_task.delay, + json.dumps( + { + "name": name, + "bucket": bucket, + } + ) + ), + using=using, + ) diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index 0d04e8db3a..757d3da9f9 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -5,6 +5,7 @@ from django.contrib import admin from django.db import models from django import forms +from django.db.models import QuerySet from rangefilter.filters import DateRangeQuickSelectListFilterBuilder from .models import (StateType, State, RelatedDocument, DocumentAuthor, Document, RelatedDocHistory, @@ -18,6 +19,9 @@ from ietf.utils.admin import SaferTabularInline from ietf.utils.validators import validate_external_resource_value +from .storage_utils import force_replication +from .utils import replicate_stored_objects_for_document + class StateTypeAdmin(admin.ModelAdmin): list_display = ["slug", "label"] @@ -73,7 +77,9 @@ class DocumentAuthorAdmin(admin.ModelAdmin): search_fields = ['document__name', 'person__name', 'email__address', 'affiliation', 'country'] raw_id_fields = ["document", "person", "email"] admin.site.register(DocumentAuthor, DocumentAuthorAdmin) - + + + class DocumentAdmin(admin.ModelAdmin): list_display = ['name', 'rev', 'group', 'pages', 'intended_std_level', 'author_list', 'time'] search_fields = ['name'] @@ -81,6 +87,7 @@ class DocumentAdmin(admin.ModelAdmin): raw_id_fields = ['group', 'shepherd', 'ad'] inlines = [DocAuthorInline, DocActionHolderInline, RelatedDocumentInline, AdditionalUrlInLine] form = DocumentForm + actions = ["replicate_stored_objects"] def save_model(self, request, obj, form, change): e = DocEvent.objects.create( @@ -95,6 +102,22 @@ def save_model(self, request, obj, form, change): def state(self, instance): return self.get_state() + @admin.action(description="Replicate related blobs") + def replicate_stored_objects(self, request, queryset: QuerySet[Document]): + doc_count = 0 + stored_obj_count = 0 + for doc in queryset.all(): + doc_count += 1 + if isinstance(doc, Document): + stored_obj_count += replicate_stored_objects_for_document(doc) + self.message_user( + request, + ( + f"Queued replication of a total of {stored_obj_count} StoredObject(s) " + f"for {doc_count} Document(s)" + ) + ) + admin.site.register(Document, DocumentAdmin) class DocHistoryAdmin(admin.ModelAdmin): @@ -232,11 +255,24 @@ class StoredObjectAdmin(admin.ModelAdmin): ] search_fields = ['name', 'doc_name', 'doc_rev'] list_display_links = ['name'] + actions = ["replicate_stored_object"] @admin.display(boolean=True, description="Deleted?", ordering="deleted") def is_deleted(self, instance): return instance.deleted is not None - + + @admin.action(description="Replicate related blobs") + def replicate_stored_object(self, request, queryset: QuerySet[StoredObject]): + stored_obj_count = 0 + for stored_object in queryset.all(): + if isinstance(stored_object, StoredObject): + force_replication(kind=stored_object.store, name=stored_object.name) + stored_obj_count += 1 + self.message_user( + request, + f"Queued replication of a total of {stored_obj_count} StoredObject(s)", + ) + admin.site.register(StoredObject, StoredObjectAdmin) diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index 9c18bb8a8a..c7cc6989cd 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -192,3 +192,12 @@ def retrieve_str(kind: str, name: str) -> str: log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}") raise return content + + +def force_replication(kind: str, name: str): + if not settings.ENABLE_BLOBSTORAGE: + return + storage = _get_storage(kind) + from ietf.blobdb.storage import BlobdbStorage + if isinstance(storage, BlobdbStorage): + storage.force_replication(name) diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 8cbe5e8f3e..6f32ed454f 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -39,12 +39,15 @@ DocHistoryAuthor, Document, DocumentAuthor, + EditedRfcAuthorsDocEvent, RfcAuthor, - State, EditedRfcAuthorsDocEvent, + State, + StoredObject, ) from ietf.doc.models import RelatedDocument, RelatedDocHistory, BallotType, DocReminder from ietf.doc.models import DocEvent, ConsensusDocEvent, BallotDocEvent, IRSGBallotDocEvent, NewRevisionDocEvent, StateDocEvent from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent, BallotPositionDocEvent +from ietf.doc.storage_utils import force_replication from ietf.name.models import DocReminderTypeName, DocRelationshipName from ietf.group.models import Role, Group, GroupFeatures from ietf.ietfauth.utils import has_role, is_authorized_in_doc_stream, is_individual_draft_author, is_bofreq_editor @@ -1713,3 +1716,23 @@ def update_or_create_draft_bibxml_file(doc, rev): def ensure_draft_bibxml_path_exists(): (Path(settings.BIBXML_BASE_PATH) / "bibxml-ids").mkdir(exist_ok=True) + + +def replicate_stored_objects_for_document(doc: Document) -> int: + """Sync all StoredObjects associated with doc to the replica blob store + + Returns count of StoredObjects queued for replication (which may or may not + be replicated, depending on whether replication is enabled / the storages are + actually BlobdbStorage instances, etc). + """ + # n.b., StoredObjects have a nullable doc_rev field, but Documents do not. + # Until / unless we straighten that out, treat "" and None equivalently when + # matching rev. + qs_matching_rev = StoredObject.objects.filter(doc_rev=doc.rev) + if doc.rev == "": + qs_matching_rev |= StoredObject.objects.filter(doc_rev__isnull=True) + count = 0 + for stored_object in qs_matching_rev.filter(doc_name=doc.name): + force_replication(kind=stored_object.store, name=stored_object.name) + count += 1 + return count From e3e080fa2341d4cae501dcfbebf8e438e49b6f46 Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Tue, 28 Apr 2026 19:02:07 +0200 Subject: [PATCH 10/17] feat: add some per country/affiliation meeting registration statistics (#10550) * Draft for meeting registrations * Add totals, + nicer JS code * Coherent URL parameters * Handle error case when per-continent stats is requested * Remove redundant code * Dynamic get the current IETF meeting * Display %-age when hovering * Add test for meeting statistics * Add test * Add statistics per affiliation * More code coverage for test * Nicer canonical affiliation * Allow navigation by buttons * Also add participants count in the legend * Add timeline over meetings (total and per country) * Default index refers to current meeting stats by number * Remove unused JS code * Add test coverage for timeline statistics * No need to import test coverage * Use stacked lines of onsite/remote when displaying the total timeline * fix a comment * Add timeline for affiliation * Expanding the test coverage to affiliation timeline * Remove unused botocore (unsure how it was added though) * Remove unused package * Code clean-up, add pan & zoom on timelines * Fix button type * refactor: avoid inline JS; safer JSON handling * chore: lint * feat: cache timeline stats Pins the top_n parameter, which had not been plumbed to be dynamically adjustable. * chore: timeout->settings + drop stale setting STATS_NAMES_LIMIT does not appear anywhere else in the codebase * refactor: wait for DOMContentLoaded + restyle * fix: fix null checks * test: update test_meeting_stats() --------- Co-authored-by: Jennifer Richards --- ietf/settings.py | 3 +- ietf/static/js/meeting_stats.js | 57 ++ ietf/static/js/meeting_timeline.js | 84 +++ ietf/stats/tests.py | 63 +- ietf/stats/urls.py | 5 +- ietf/stats/views.py | 612 ++++++++++++++++++-- ietf/templates/base/menu.html | 11 +- ietf/templates/stats/index.html | 8 +- ietf/templates/stats/meeting_stats.html | 61 ++ ietf/templates/stats/meetings_timeline.html | 78 +++ package.json | 2 + 11 files changed, 916 insertions(+), 68 deletions(-) create mode 100644 ietf/static/js/meeting_stats.js create mode 100644 ietf/static/js/meeting_timeline.js create mode 100644 ietf/templates/stats/meeting_stats.html create mode 100644 ietf/templates/stats/meetings_timeline.html diff --git a/ietf/settings.py b/ietf/settings.py index 3aa45a453c..50e069ff1a 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -231,6 +231,7 @@ AGENDA_CACHE_TIMEOUT_DEFAULT = 8 * 24 * 60 * 60 # 8 days AGENDA_CACHE_TIMEOUT_CURRENT_MEETING = 6 * 60 # 6 minutes + WSGI_APPLICATION = "ietf.wsgi.application" AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', ) @@ -1270,7 +1271,7 @@ def skip_unreadable_post(record): except ImportError: pass -STATS_NAMES_LIMIT = 25 +STATS_TIMELINE_CACHE_TIMEOUT = 86400 UTILS_MEETING_CONFERENCE_DOMAINS = ['webex.com', 'zoom.us', 'jitsi.org', 'meetecho.com', 'gather.town', ] UTILS_TEST_RANDOM_STATE_FILE = '.factoryboy_random_state' diff --git a/ietf/static/js/meeting_stats.js b/ietf/static/js/meeting_stats.js new file mode 100644 index 0000000000..70b18a0f03 --- /dev/null +++ b/ietf/static/js/meeting_stats.js @@ -0,0 +1,57 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +document.addEventListener('DOMContentLoaded', () => { + // Need to use autocolors plug-in else all slices are gray... + const autocolors = window['chartjs-plugin-autocolors'] + Chart.register(autocolors) + // ── Safely parse JSON data injected from Django view ── + const totalChartData = JSON.parse(document.getElementById('total-chart-data').textContent) + const inPersonChartData = JSON.parse(document.getElementById('in-person-chart-data').textContent) + + function displayChart (id, data) { + const ctx = document.getElementById(id).getContext('2d') + new Chart(ctx, { + type: 'pie', // Change to 'doughnut' for a donut chart + data: data, + options: { + responsive: true, + plugins: { + autocolors: { + mode: 'data' // Required for Pie charts to color individual slices + }, + legend: { + position: 'bottom', + labels: { + padding: 20, + font: { size: 13 }, + color: '#475569', + generateLabels: function (chart) { + const dataset = chart.data.datasets[0] + return chart.data.labels.map((label, i) => ({ + text: `${label}: ${dataset.data[i]}`, + fillStyle: dataset.backgroundColor[i], + hidden: false, + index: i, + })) + } + } + }, + tooltip: { + callbacks: { + label: function (context) { + const label = context.label || '' + const value = context.raw + const total = context.dataset.data.reduce((a, b) => a + b, 0) + const percentage = ((value / total) * 100).toFixed(1) + + return `${label}: ${value} (${percentage}%)` + } + } + } + } + } + }) + } + + displayChart('totalRegistrationChart', totalChartData) + displayChart('inPersonRegistrationChart', inPersonChartData) +}) diff --git a/ietf/static/js/meeting_timeline.js b/ietf/static/js/meeting_timeline.js new file mode 100644 index 0000000000..161cead0ec --- /dev/null +++ b/ietf/static/js/meeting_timeline.js @@ -0,0 +1,84 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +document.addEventListener('DOMContentLoaded', () => { + // ── Safely parse JSON data injected from Django view ── + const totalChartData = JSON.parse(document.getElementById('total-chart-data').textContent) + const inPersonChartData = JSON.parse(document.getElementById('in-person-chart-data').textContent) + const statsType = JSON.parse(document.getElementById('stats-type-data').textContent) + const stackedLines = statsType === 'total' + + function displayChart (id, data) { + const ctx = document.getElementById(id).getContext('2d') + return new Chart(ctx, { + type: 'line', // Change to 'doughnut' for a donut chart + data: data, + options: { + responsive: true, + scales: { + y: { + stacked: stackedLines, + }, + x: { + title: { + display: true, + text: 'IETF Meeting Number', + }, + }, + }, + plugins: { + legend: { + position: 'bottom', + labels: { + usePointStyle: true, + padding: 15, + font: { size: 12 }, + }, + }, + tooltip: { + backgroundColor: 'rgba(0,0,0,0.8)', + titleFont: { size: 14 }, + bodyFont: { size: 13 }, + callbacks: { + title: function (items) { + return `IETF Meeting ${items[0].label}` + }, + label: function (context) { + return ` ${context.dataset.label}: ${context.parsed.y} participants` + } + } + }, + zoom: { + zoom: { + wheel: { enabled: true }, // scroll to zoom + pinch: { enabled: true }, // pinch on mobile + drag: { enabled: true }, // drag to select range + mode: 'xy', // zoom X-axis and Y-axis + }, + pan: { + enabled: true, + mode: 'xy', // pan X-axis and Y-axis + }, + }, + } + } + }) + } + + const totalChart = displayChart('totalRegistrationChart', totalChartData) + if (inPersonChartData !== null) { + inPersonChart = displayChart('inPersonRegistrationChart', inPersonChartData) + } + document.addEventListener('keydown', (event) => { + if (event.key === 'Escape') { + totalChart.resetZoom() + if (inPersonChart !== null) { + inPersonChart.resetZoom() + } + } + }) + document.getElementById('resetButton').addEventListener('click', () => { + totalChart.resetZoom() + if (inPersonChart !== null) { + inPersonChart.resetZoom() + } + }) +}) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 48552c8fba..373f06e343 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -4,12 +4,14 @@ import calendar import json +import datetime from pyquery import PyQuery import debug # pyflakes:ignore from django.urls import reverse as urlreverse +from django.utils import timezone from ietf.utils.test_utils import login_testing_unauthorized, TestCase import ietf.stats.views @@ -18,24 +20,73 @@ from ietf.group.factories import RoleFactory from ietf.person.factories import PersonFactory from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory +from ietf.meeting.tests_models import MeetingFactory, RegistrationFactory from ietf.utils.timezone import date_today class StatisticsTests(TestCase): def test_stats_index(self): + # Create a meeting as the index page needs to know the current meeting + MeetingFactory(type_id='ietf', number='124', date=timezone.now()) url = urlreverse(ietf.stats.views.stats_index) r = self.client.get(url) self.assertEqual(r.status_code, 200) def test_document_stats(self): - r = self.client.get(urlreverse("ietf.stats.views.document_stats")) - self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index")) - + # Create a meeting as the index page needs to know the current meeting + MeetingFactory(type_id='ietf', number='124', date=timezone.now()) + r = self.client.get(urlreverse(ietf.stats.views.document_stats)) + self.assertRedirects(r, urlreverse(ietf.stats.views.stats_index)) def test_meeting_stats(self): - r = self.client.get(urlreverse("ietf.stats.views.meeting_stats")) - self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index")) - + meeting124 = MeetingFactory(type_id='ietf', number='124', date=timezone.now()) + meeting125 = MeetingFactory(type_id='ietf', number='125', date=timezone.now() + datetime.timedelta(days=120)) + RegistrationFactory.create_batch(15, meeting=meeting124, with_ticket={'attendance_type_id': 'onsite'}, attended=True) + RegistrationFactory(meeting=meeting124, with_ticket={'attendance_type_id': 'onsite'}, attended=False) + RegistrationFactory.create_batch(14, meeting=meeting124, with_ticket={'attendance_type_id': 'remote'}, attended=True) + RegistrationFactory(meeting=meeting124, with_ticket={'attendance_type_id': 'remote'}, attended=False) + RegistrationFactory.create_batch(15, meeting=meeting125, affiliation='Test LLC', with_ticket={'attendance_type_id': 'remote'}, attended=False) + RegistrationFactory.create_batch(25, meeting=meeting125, affiliation='Example, Ltd', with_ticket={'attendance_type_id': 'onsite'}, attended=False) + # Test the meeting specific statitistics per affiliation and per country + r = self.client.get(urlreverse(ietf.stats.views.meeting_stats, kwargs={"meeting_number": "124", "stats_type": "affiliation"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Total Registrations by Affiliation (31 in total)") + self.assertContains(r, "In Person Registrations by Affiliation (16 in total)") + self.assertContains(r, "/stats/meeting/124/affiliation") + self.assertContains(r, "/stats/meeting/125/affiliation") + r = self.client.get(urlreverse(ietf.stats.views.meeting_stats, kwargs={"meeting_number": "124", "stats_type": "country"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Total Registrations by Country (31 in total)") + self.assertContains(r, "In Person Registrations by Country (16 in total)") + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + # Test the meetings timeline per country + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "country"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + self.assertContains(r, "This page provides a timeline of meeting registrations by country") + # Test the meetings timeline per affiliation + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "affiliation"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/affiliation") + self.assertContains(r, "/stats/meeting/125/affiliation") + self.assertContains(r, "This page provides a timeline of meeting registrations by affiliation") + # Extract the JSON embedded in the response + pq = PyQuery(r.content) + in_person_data = json.loads(pq.find("script#in-person-chart-data").text()) + self.assertTrue( + any( + ds["label"] == "Example" and ds["data"] == [0, 25] + for ds in in_person_data["datasets"] + ) + ) + # Test the global meetings timeline + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "total"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + self.assertContains(r, "This page provides a timeline of meeting registrations.") def test_known_country_list(self): # check redirect diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index d2993759d2..01b8758c84 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -11,7 +11,8 @@ url(r"^$", views.stats_index), url(r"^document/(?:(?Pauthors|pages|words|format|formlang|author/(?:documents|affiliation|country|continent|citations|hindex)|yearly/(?:affiliation|country|continent))/)?$", views.document_stats), url(r"^knowncountries/$", views.known_countries_list), - url(r"^meeting/(?P\d+)/(?Pcountry|continent)/$", views.meeting_stats), - url(r"^meeting/(?:(?Poverview|country|continent)/)?$", views.meeting_stats), + url(r"^meeting/$", views.meetings_timeline), + url(r"^meeting/(?P\d+)/(?Paffiliation|country)/$", views.meeting_stats), + url(r"^meeting/(?:(?Paffiliation|country|total)/)?$", views.meetings_timeline), url(r"^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), ] diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 504d84e86d..d61b673075 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -9,11 +9,13 @@ import dateutil.relativedelta from collections import defaultdict +from django.conf import settings from django.contrib.auth.decorators import login_required +from django.core.cache import cache from django.http import HttpResponseRedirect from django.shortcuts import render from django.urls import reverse as urlreverse - +from django.db.models import Count import debug # pyflakes:ignore @@ -25,15 +27,32 @@ from ietf.group.models import Role, Group from ietf.person.models import Person from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName +from ietf.meeting.models import Registration from ietf.ietfauth.utils import has_role from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, DEADLINE_TZINFO +from ietf.meeting.helpers import get_current_ietf_meeting_num, get_ietf_meeting +# Color palette for lines +colors = [ + '#FF6384', '#36A2EB', '#FFCE56', '#4BC0C0', '#9966FF', + '#FF9F40', '#C9CBCF', '#7BC043', '#F37735', '#00ABA9', + '#2B5797', '#E81123', '#00A4EF', '#7FBA00', '#FFB900', + '#D83B01', '#B4009E', '#5C2D91', '#008575', '#E3008C', +] def stats_index(request): - return render(request, "stats/index.html") + """Render the statistics index page with the current meeting number as it is required by the meeting menu item.""" + current_meeting = get_current_ietf_meeting_num() + return render(request, "stats/index.html", { + "current_meeting": current_meeting + }) def generate_query_string(query_dict, overrides): + """ + Returns: + A query string starting with '?' if there are parameters, empty string otherwise. + """ query_part = "" if query_dict or overrides: @@ -58,9 +77,20 @@ def generate_query_string(query_dict, overrides): return query_part def get_choice(request, get_parameter, possible_choices, multiple=False): - # the statistics are built with links to make navigation faster, - # so we don't really have a form in most cases, so just use this - # helper instead to select between the choices + """Extract a choice from the request GET parameters. + + Since statistics pages use links for navigation instead of forms, + this helper selects between possible choices from the URL parameters. + + Args: + request: The HTTP request object. + get_parameter: The name of the GET parameter. + possible_choices: List of tuples (value, label). + multiple: If True, return a list of found values; otherwise return the first found or None. + + Returns: + The selected value(s) or None. + """ values = request.GET.getlist(get_parameter) found = [t[0] for t in possible_choices if t[0] in values] @@ -73,75 +103,553 @@ def get_choice(request, get_parameter, possible_choices, multiple=False): return None def add_url_to_choices(choices, url_builder): + """Add URLs to a list of choices. + + Args: + choices: List of tuples (slug, label). + url_builder: Function that takes a slug and returns a URL. + + Returns: + List of tuples (slug, label, url). + """ return [ (slug, label, url_builder(slug)) for slug, label in choices] -def put_into_bin(value, bin_size): - if value is None: - return (0, '') +def document_stats(request, stats_type=None): + # timeline per year, or per specific year: streams, affiliation, rfc vs I-D + # could also be time between individual/WG I-D to rfc publication/IESG ballot + # DISCUSS resolution time + # Humm also split by authors (affiliation) / documents (the rest) probably + """Redirect to the stats index page. Deprecated view.""" + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) - v = (value // bin_size) * bin_size - return (v, "{} - {}".format(v, v + bin_size - 1)) +def known_countries_list(request, stats_type=None, acronym=None): + """Render a list of known countries with their aliases.""" + countries = CountryName.objects.prefetch_related("countryalias_set") + for c in countries: + # the sorting is a bit of a hack - it puts the ISO code first + # since it was added in a migration + c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) -def prune_unknown_bin_with_known(bins): - # remove from the unknown bin all authors within the - # named/known bins - all_known = { n for b, names in bins.items() if b for n in names } - bins[""] = [name for name in bins[""] if name not in all_known] - if not bins[""]: - del bins[""] + return render(request, "stats/known_countries_list.html", { + "countries": countries, + }) -def count_bins(bins): - return len({ n for b, names in bins.items() if b for n in names }) +def canonicalize_affiliation(affiliation): + """Canonicalize an affiliation string by removing common suffixes and standardizing prefixes. + + Args: + affiliation: The affiliation string to canonicalize. + + Returns: + The canonicalized affiliation string, or None if input is None. + """ + if not affiliation or affiliation.lower() in ('n/a', 'none', 'unspecified'): + return None + for suffix in ('ab', 'ag', 'corp', 'corp.', 'corporation', 'gmbh', 'inc.', 'inc', 'international pte ltd', 'llc', 'ltd', 'ltd.', 'private limited', 'pty ltd', 'pvt ltd'): + if affiliation.lower().endswith(', ' + suffix): + affiliation = affiliation[:-(len(suffix)+2)] + elif affiliation.lower().endswith(' ' + suffix): + affiliation = affiliation[:-(len(suffix)+1)] + elif affiliation.lower().endswith(',' + suffix): + affiliation = affiliation[:-(len(suffix)+1)] + for prefix in ('akamai','apple', 'cisco', 'futurewei', 'google', 'hitachi', 'hpe', 'huawei', 'juniper', 'meta', 'nokia', 'ntt', 'siemens'): + if affiliation.lower().startswith(prefix + ' '): + affiliation = prefix + return affiliation.title() + +def get_affiliation_data_for_meetings(attendance_type=None): + """Get affiliation participation data for meetings timeline chart. + + Args: + attendance_type: Optional filter for attendance type (e.g., 'onsite'). + + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = f'stats:get_affiliation_data_for_meetings:{attendance_type}' + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + top_n = 20 # could be a parameter, but would need to adjust cache handling + + # Get registration status details + if attendance_type: + registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) + else: + registrations = Registration.objects.all() + registrations = registrations.values('affiliation', 'meeting__number') + + # Count per canonicalized affiliation + organization = dict() + meetings_set = set() + org_totals = defaultdict(int) + data_map = defaultdict(dict) # {org: {meeting: count}} + + for reg in registrations: + meeting = reg['meeting__number'] + meetings_set.add(meeting) + affiliation = canonicalize_affiliation(reg['affiliation']) or "Unspecified" + organization[affiliation] = organization.get(affiliation, 0) + 1 + org_totals[affiliation] = org_totals.get(affiliation, 0) + 1 + data_map[affiliation][meeting] = data_map[affiliation].get(meeting, 0) + 1 + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + + # ── Step 3: Get top N countries ── + top_orgs = sorted( + org_totals.keys(), + key=lambda c: org_totals[c], + reverse=True + )[:top_n] + non_top_orgs = org_totals.keys() - top_orgs + other_totals = defaultdict(int) + for m in sorted_meetings: + other_totals[m] = 0 + for c in non_top_orgs: + other_totals[m] += int(data_map[c].get(m, 0)) + + # ── Step 4: Build Chart.js datasets ── + + datasets = [] + for idx, org in enumerate(top_orgs): + color = colors[idx % len(colors)] + datasets.append({ + 'label': org, + 'data': [data_map[org].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'fill': False, + 'tension': 0.3, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + + # -- Step 4.bis handle the other -- + datasets.append({ + 'label': 'Other', + 'data': [other_totals.get(m, 0) for m in sorted_meetings], + 'borderColor': 'black', + 'fill': False, + 'tension': 0.3, + 'pointColor': 'black', + 'pointBackgroundColor': 'black', + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) -def add_labeled_top_series_from_bins(chart_data, bins, limit): - """Take bins on the form (x, label): [name1, name2, ...], figure out - how many there are per label, take the overall top ones and put - them into sorted series like [(x1, len(names1)), (x2, len(names2)), ...].""" - aggregated_bins = defaultdict(set) - xs = set() - for (x, label), names in bins.items(): - xs.add(x) - aggregated_bins[label].update(names) + return sorted_meetings, datasets - xs = list(sorted(xs)) +def get_country_data_for_meetings(attendance_type=None): + """Get country participation data for meetings timeline chart. - sorted_bins = sorted(aggregated_bins.items(), key=lambda t: len(t[1]), reverse=True) - top = [ label for label, names in list(sorted_bins)[:limit]] + Args: + attendance_type: Optional filter for attendance type (e.g., 'onsite'). - for label in top: - series_data = [] + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = f'stats:get_country_data_for_meetings:{attendance_type}' + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + top_n = 10 # could be a parameter, but would need to adjust cache handling + # Get registration status counts, aggregated by country_code + if attendance_type: + registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) + else: + registrations = Registration.objects.all() + queryset = ( + registrations + .values( + 'meeting__number', # e.g. "118", "119", "120" + 'country_code' # country code of the participant + ) + .annotate(participant_count=Count('id')) + .order_by('meeting__number') # chronological order + ) + + # ── Step 1: Collect all meetings and country totals ── + meetings_set = set() + country_totals = defaultdict(int) + data_map = defaultdict(dict) # {country: {meeting: count}} + + for row in queryset: + meeting = row['meeting__number'] + country = row['country_code'] + count = row['participant_count'] + + meetings_set.add(meeting) + country_totals[country] += count + data_map[country][meeting] = count + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + + # ── Step 3: Get top N countries ── + top_countries = sorted( + country_totals.keys(), + key=lambda c: country_totals[c], + reverse=True + )[:top_n] + + # -- Step 3.bis do the 'other' category -- + non_top_countries = country_totals.keys() - top_countries + other_totals = defaultdict(int) + for m in sorted_meetings: + other_totals[m] = 0 + for c in non_top_countries: + other_totals[m] += int(data_map[c].get(m, 0)) + + # ── Step 4: Build Chart.js datasets ── + + datasets = [] + for idx, country in enumerate(top_countries): + color = colors[idx % len(colors)] + datasets.append({ + 'label': country, + 'data': [data_map[country].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'fill': False, + 'tension': 0.3, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + + # -- Step 4.bis handle the other -- + datasets.append({ + 'label': 'Other', + 'data': [other_totals.get(m, 0) for m in sorted_meetings], + 'borderColor': 'black', + 'fill': False, + 'tension': 0.3, + 'pointColor': 'black', + 'pointBackgroundColor': 'black', + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) - for x in xs: - names = bins.get((x, label), set()) + return sorted_meetings, datasets + +def get_data_for_meetings(): + """Get total participation data by attendance type for meetings timeline chart. + + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = "stats:get_data_for_meetings" + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + # Get registration status counts, aggregated by ticket types + registrations = Registration.objects.filter(tickets__attendance_type__in=['onsite', 'remote']) + queryset = ( + registrations + .values( + 'meeting__number', # e.g. "118", "119", "120" + 'tickets__attendance_type' + ) + .annotate(participant_count=Count('id')) + .order_by('meeting__number') # chronological order + ) + + # ── Step 1: Collect all meetings and tickets totals ── + meetings_set = set() + tickets_totals = defaultdict(int) + data_map = defaultdict(dict) # {ticket: {meeting: count}} + + for row in queryset: + meeting = row['meeting__number'] + ticket = row['tickets__attendance_type'] + count = row['participant_count'] + + meetings_set.add(meeting) + tickets_totals[ticket] += count + data_map[ticket][meeting] = count + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + ticket_types = tickets_totals.keys() + + # ── Step 4: Build Chart.js datasets ── + # Color palette for lines + colors = [ '#FF6384', '#36A2EB'] + + datasets = [] + for idx, ticket_type in enumerate(ticket_types): + color = colors[idx % len(colors)] + datasets.append({ + 'label': ticket_type, + 'data': [data_map[ticket_type].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'backgroundColor': color + '99', # 60% opacity fill + 'fill': True, + 'tension': 0.0, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) + return sorted_meetings, datasets + +def meetings_timeline(request, stats_type='country'): + """Render the meetings timeline page with participation statistics over time. + + Args: + request: The HTTP request object. + stats_type: Type of statistics ('country' or 'total'). + top_n: Number of top items to show (for country stats). + + Returns: + Rendered response for the meetings timeline template. + """ + if stats_type == 'total': + total_labels, total_data_sets = get_data_for_meetings() + in_person_labels = ([], []) + in_person_data_sets = ([], []) + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + elif stats_type == 'affiliation': + total_labels, total_data_sets = get_affiliation_data_for_meetings() + in_person_labels, in_person_data_sets = get_affiliation_data_for_meetings(attendance_type='onsite') + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + elif stats_type == 'country': + total_labels, total_data_sets = get_country_data_for_meetings() + in_person_labels, in_person_data_sets = get_country_data_for_meetings(attendance_type='onsite') + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) - series_data.append((x, len(names))) + total_chart_data = { + 'labels': total_labels, + 'datasets': total_data_sets, + } - chart_data.append({ - "data": series_data, - "name": label - }) + # On per country/affiliation have a separate graph for inperson + if stats_type == 'total': + in_person_chart_data = None + else: + in_person_chart_data = { + 'labels': in_person_labels, + 'datasets': in_person_data_sets, + } -def document_stats(request, stats_type=None): - return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + # Prepare the list of choice buttons for the template + possible_stats_types = [ + ("affiliation", "Per affiliation", urlreverse(meetings_timeline, kwargs={'stats_type': 'affiliation'})), + ("country", "Per country", urlreverse(meetings_timeline, kwargs={'stats_type': 'country'})), + ("total", "Total", urlreverse(meetings_timeline, kwargs={'stats_type': 'total'})), + ] + current_meeting = get_current_ietf_meeting_num() + if stats_type == 'total': + possible_stats_type = 'country' + else: + possible_stats_type = stats_type -def known_countries_list(request, stats_type=None, acronym=None): - countries = CountryName.objects.prefetch_related("countryalias_set") - for c in countries: - # the sorting is a bit of a hack - it puts the ISO code first - # since it was added in a migration - c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) + possible_meeting_numbers = [(int(current_meeting)-1, urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting)-1, 'stats_type': possible_stats_type})), + (int(current_meeting), urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting), 'stats_type': possible_stats_type})), + (int(current_meeting)+1, urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting)+1, 'stats_type': possible_stats_type}))] - return render(request, "stats/known_countries_list.html", { - "countries": countries, + return render(request, "stats/meetings_timeline.html", { + "top_n": top_n, + "possible_stats_types": possible_stats_types, + "possible_meeting_numbers": possible_meeting_numbers, + "stats_type": stats_type, + "total_chart_data": total_chart_data, + "in_person_chart_data": in_person_chart_data, }) -def meeting_stats(request, num=None, stats_type=None): - return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) +def get_affiliation_data_for_meeting(meeting_number, minimum_required, attendance_type=None): + """Get affiliation participation data for a specific meeting. + + Args: + meeting_number: The meeting number. + minimum_required: Minimum count to include in main data (others go to 'Other'). + attendance_type: Optional filter for attendance type. + + Returns: + Tuple of (labels, data, total) for chart display. + """ + # Get registration status details + registrations = Registration.objects.filter(meeting__number=meeting_number) + if attendance_type: + registrations = registrations.filter(tickets__attendance_type=attendance_type) + registrations = registrations.values('affiliation') + + # Count per canonicalized affiliation + organization = dict() + for reg in registrations: + affiliation = canonicalize_affiliation(reg['affiliation']) or "Unspecified" + organization[affiliation] = organization.get(affiliation, 0) + 1 + + # Sort to have the largest count first (nicer in pie chart) + sorted_orgs = sorted(organization.items(), key=lambda t: t[1], reverse=True) + labels = [] + data = [] + others_count = 0 + total = 0 + for org, count in sorted_orgs: + total += count + if count > minimum_required: + labels.append(org) + data.append(count) + else: + others_count += count + + if others_count > 0: + labels.append('Other') + data.append(others_count) + + return labels, data, total + +def get_data_for_meeting(meeting_number, minimum_required, attendance_type=None): + """Get country participation data for a specific meeting. + + Args: + meeting_number: The meeting number. + minimum_required: Minimum count to include in main data (others go to 'Other'). + attendance_type: Optional filter for attendance type. + + Returns: + Tuple of (labels, data, total) for chart display. + """ + # Get registration status counts, aggregated by country_code + registration_counts = Registration.objects.filter(meeting__number=meeting_number) + if attendance_type: + registration_counts = registration_counts.filter(tickets__attendance_type=attendance_type) + registration_counts = registration_counts.values('country_code').annotate(count=Count('country_code')).order_by('-count') + + labels = [] + data = [] + others_count = 0 + total = 0 + for item in registration_counts: + total += item['count'] + if item['count'] > minimum_required: + labels.append(item['country_code']) + data.append(item['count']) + else: + others_count += item['count'] + + if others_count > 0: + labels.append('Other') + data.append(others_count) + + return labels, data, total + +def meeting_stats(request, meeting_number=None, stats_type='country'): + """Render statistics for a specific meeting. + + Args: + request: The HTTP request object. + meeting_number: The meeting number (defaults to current). + stats_type: Type of statistics ('country' or 'affiliation'). + + Returns: + Rendered response for the meeting stats template. + """ + + current_meeting = get_current_ietf_meeting_num() + if meeting_number is None: + meeting_number = current_meeting + + this_meeting = get_ietf_meeting(meeting_number) + + if stats_type == 'affiliation': + minimum_required = 4 + total_labels, total_data, total_total = get_affiliation_data_for_meeting(meeting_number, minimum_required) + in_person_labels, in_person_data, in_person_total = get_affiliation_data_for_meeting(meeting_number, minimum_required, attendance_type='onsite') + elif stats_type == 'country': + minimum_required = 10 + total_labels, total_data, total_total = get_data_for_meeting(meeting_number, minimum_required) + in_person_labels, in_person_data, in_person_total = get_data_for_meeting(meeting_number, minimum_required, attendance_type='onsite') + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + + total_chart_data = { + 'labels': total_labels, + 'datasets': [{ + 'label': 'Total Registrations by ' + stats_type, + 'data': total_data, + 'borderColor': '#ffffff', + 'borderWidth': 2, + }] + } + in_person_chart_data = { + 'labels': in_person_labels, + 'datasets': [{ + 'label': 'In Person Registrations by ' + stats_type, + 'data': in_person_data, + 'borderColor': '#ffffff', + 'borderWidth': 2, + }] + } + + # Prepare the list of choice buttons for the template + possible_stats_types = [ + ("affiliation", "Per affiliation", urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': 'affiliation'})), + ("country", "Per country", urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': 'country'})), + ] + + # Prepare the list of meeting number buttons for the template + possible_meeting_numbers = [('All', urlreverse(meetings_timeline, kwargs={'stats_type': stats_type}))] + if int(meeting_number) > 72: # No registration data before IETF-72 + possible_meeting_numbers.append((int(meeting_number)-1, urlreverse(meeting_stats, kwargs={'meeting_number': int(meeting_number)-1, 'stats_type': stats_type}))) + possible_meeting_numbers.append((meeting_number, urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': stats_type}))) + if int(meeting_number) <= int(current_meeting): # Allow current meeting +1 + possible_meeting_numbers.append((int(meeting_number)+1, urlreverse(meeting_stats, kwargs={'meeting_number': int(meeting_number)+1, 'stats_type': stats_type}))) + + return render(request, "stats/meeting_stats.html", { + "meeting_number": meeting_number, + "meeting_date": this_meeting.date, + "meeting_country": this_meeting.country, + "meeting_city": this_meeting.city, + "possible_stats_types": possible_stats_types, + "possible_meeting_numbers": possible_meeting_numbers, + "stats_type": stats_type, + "minimum_required": minimum_required, + "total_chart_data": total_chart_data, + "total_total": total_total, + "in_person_chart_data": in_person_chart_data, + "in_person_total": in_person_total + }) @login_required def review_stats(request, stats_type=None, acronym=None): + """Render review statistics page with tables and charts for review assignments. + + Shows completion status, results, assignment states, and time series data. + Supports both team-level and reviewer-level views with filtering options. + + Args: + request: The HTTP request object. + stats_type: Type of statistics ('completion', 'results', 'states', 'time'). + acronym: Team acronym for reviewer-level view (None for team view). + + Returns: + Rendered response for the review stats template. + """ # This view is a bit complex because we want to show a bunch of # tables with various filtering options, and both a team overview # and a reviewers-within-team overview - and a time series chart. diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html index 8ff6e952da..43ca025e28 100644 --- a/ietf/templates/base/menu.html +++ b/ietf/templates/base/menu.html @@ -428,12 +428,11 @@ Downref registry -
  • - +
  • + Statistics -
      +

      - Statistics on meetings and authorship are not currently available. + Statistics on authorship are not currently available.

      {% endblock %} \ No newline at end of file diff --git a/ietf/templates/stats/meeting_stats.html b/ietf/templates/stats/meeting_stats.html new file mode 100644 index 0000000000..b32f5a4046 --- /dev/null +++ b/ietf/templates/stats/meeting_stats.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} +{% load origin %} +{% origin %} +{% load ietf_filters static django_bootstrap5 %} +{% block js %} + + + + {{ total_chart_data|json_script:"total-chart-data" }} + {{ in_person_chart_data|json_script:"in-person-chart-data" }} + +{% endblock %} +{% block content %} + {% origin %} +

      + {% block title %} + Statistics for IETF-{{ meeting_number }} ({{ meeting_date }}, {{ meeting_city }}, {{ meeting_country }}) Registrations + {% endblock %} +

      +
      + +
      + {% for slug, label, url in possible_stats_types %} + {{ label }} + {% endfor %} +
      + +
      + {% for num, url in possible_meeting_numbers %} + {{ num }} + {% endfor %} +
      +
      +

      + This page provides a visual representation of the total registrations for IETF-{{ meeting_number }} by {{ stats_type }}. + Only categories having more than {{ minimum_required }} registrations are displayed separately, + else they are grouped under "Other". +

      +
      +
      +

      Total Registrations by {{ stats_type|title }} ({{ total_total}} in total)

      +
      + +
      +
      +
      +

      In Person Registrations by {{ stats_type|title }} ({{ in_person_total}} in total)

      +
      + +
      +
      +
      +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/stats/meetings_timeline.html b/ietf/templates/stats/meetings_timeline.html new file mode 100644 index 0000000000..65fb4e09c0 --- /dev/null +++ b/ietf/templates/stats/meetings_timeline.html @@ -0,0 +1,78 @@ +{% extends "base.html" %} +{% load origin %} +{% origin %} +{% load ietf_filters static django_bootstrap5 %} +{% block js %} + + + + + {{ total_chart_data|json_script:"total-chart-data" }} + {{ in_person_chart_data|json_script:"in-person-chart-data" }} + {{ stats_type|json_script:"stats-type-data" }} + +{% endblock %} +{% block content %} + {% origin %} +

      + {% block title %} + Statistics for IETF Meeting Registrations + {% endblock %} +

      + +
      + +
      + {% for slug, label, url in possible_stats_types %} + {{ label }} + {% endfor %} +
      + +
      + {% for num, url in possible_meeting_numbers %} + {{ num }} + {% endfor %} +
      +
      +

      + {% if stats_type == 'total' %} + This page provides a timeline of meeting registrations. + {% else %} + This page provides a timeline of meeting registrations by {{ stats_type }} with a limit of {{ top_n }} categories. + {% endif %} + Panning can be done via the mouse or with a finger. Zooming is done via the mouse wheel or via a pinch gesture. Press ESC + or click to reset panning/zooming. +

      +
      +
      + {% if stats_type == 'total' %} +

      Total Registrations

      + {% else %} +

      Total Registrations by {{ stats_type|title }}

      + {% endif %} +
      + +
      +
      + {% if stats_type != 'total' %} +
      + {% if stats_type == 'total' %} +

      Total In Person Registrations

      + {% else %} +

      In Person Registrations by {{ stats_type|title }}

      + {% endif %} +
      + +
      +
      + {% endif %} +
      +{% endblock %} \ No newline at end of file diff --git a/package.json b/package.json index 57642d7860..6f61aaba29 100644 --- a/package.json +++ b/package.json @@ -146,6 +146,8 @@ "ietf/static/js/manage-community-list.js", "ietf/static/js/manage-review-requests.js", "ietf/static/js/meeting-interim-request.js", + "ietf/static/js/meeting_stats.js", + "ietf/static/js/meeting_timeline.js", "ietf/static/js/moment.js", "ietf/static/js/navbar-doc-search.js", "ietf/static/js/password_strength.js", From 4964901bb5a29d37ab7a8f7ee925eda7cf734b6f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Apr 2026 18:53:12 -0300 Subject: [PATCH 11/17] fix: bind gunicorn on ipv6 interface + config refactor (#10769) * refactor: move gunicorn cfg to python * fix: bind gunicorn on ipv6 interface, too --- dev/build/datatracker-start.sh | 12 +----------- dev/build/gunicorn.conf.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index a676415a26..012a563412 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -45,16 +45,6 @@ cleanup () { trap 'trap "" TERM; cleanup' TERM # start gunicorn in the background so we can trap the TERM signal -gunicorn \ - -c /workspace/gunicorn.conf.py \ - --workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \ - --max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \ - --timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \ - --bind :8000 \ - --log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \ - --capture-output \ - --access-logfile -\ - ${DATATRACKER_GUNICORN_EXTRA_ARGS} \ - ietf.wsgi:application & +gunicorn -c /workspace/gunicorn.conf.py ${DATATRACKER_GUNICORN_EXTRA_ARGS} ietf.wsgi:application & gunicorn_pid=$! wait "${gunicorn_pid}" diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py index 9af4478685..37991a55dc 100644 --- a/dev/build/gunicorn.conf.py +++ b/dev/build/gunicorn.conf.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2024-2025, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved import os import ietf @@ -12,6 +12,22 @@ from opentelemetry.instrumentation.pymemcache import PymemcacheInstrumentor from opentelemetry.instrumentation.requests import RequestsInstrumentor +# Bind interfaces for both ipv4 and ipv6 +bind = ["127.0.0.1:8000", "[::1]:8000"] + +# Disable control socket +control_socket_disable = True + +# Settings configurable via environment +workers = int(os.environ.get("DATATRACKER_GUNICORN_WORKERS", "9")) +max_requests = int(os.environ.get("DATATRACKER_GUNICORN_MAX_REQUESTS", "32768")) +timeout = int(os.environ.get("DATATRACKER_GUNICORN_TIMEOUT", "180")) +loglevel = os.environ.get("DATATRACKER_GUNICORN_LOG_LEVEL", "info") + +# Logging / stdout capture +capture_output = True +accesslog = "-" + # Configure security scheme headers for forwarded requests. Cloudflare sets X-Forwarded-Proto # for us. Don't trust any of the other similar headers. Only trust the header if it's coming # from localhost, as all legitimate traffic will reach gunicorn via co-located nginx. From e19761d6043f09883c1989b62334fb8837390581 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Apr 2026 21:43:37 -0300 Subject: [PATCH 12/17] fix: bind gunicorn on all interfaces (#10774) * fix: bind gunicorn on all interfaces * fix: something conflicts with ipv6 - loopback only * fix: bind all ipv6 (conflict is in dev only) --- dev/build/gunicorn.conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py index 37991a55dc..18f6d0aad1 100644 --- a/dev/build/gunicorn.conf.py +++ b/dev/build/gunicorn.conf.py @@ -12,8 +12,8 @@ from opentelemetry.instrumentation.pymemcache import PymemcacheInstrumentor from opentelemetry.instrumentation.requests import RequestsInstrumentor -# Bind interfaces for both ipv4 and ipv6 -bind = ["127.0.0.1:8000", "[::1]:8000"] +# Bind all ipv4 and ipv6 interfaces +bind = ["0.0.0.0:8000", "[::]:8000"] # Disable control socket control_socket_disable = True From 7649d1c4b6f5cc669dd70d4512042a867c2081e1 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Apr 2026 22:15:11 -0300 Subject: [PATCH 13/17] fix: bind gunicorn to ipv6 loopback only (#10776) Something conflicts with [::]:8000... --- dev/build/gunicorn.conf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py index 18f6d0aad1..be8808ec48 100644 --- a/dev/build/gunicorn.conf.py +++ b/dev/build/gunicorn.conf.py @@ -12,8 +12,9 @@ from opentelemetry.instrumentation.pymemcache import PymemcacheInstrumentor from opentelemetry.instrumentation.requests import RequestsInstrumentor -# Bind all ipv4 and ipv6 interfaces -bind = ["0.0.0.0:8000", "[::]:8000"] +# Bind all ipv4 interfaces and ipv6 loopback interface. Would prefer to bind all +# ipv6 as well, but something conflicts with [::]:8000. +bind = ["0.0.0.0:8000", "[::1]:8000"] # Disable control socket control_socket_disable = True From d9196b53cfa5d0034ddb9aae7d4e40f91404a3b5 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Thu, 30 Apr 2026 02:56:00 +1200 Subject: [PATCH 14/17] fix: Disable calt font feature (#10779) --- ietf/static/css/ietf.scss | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ietf/static/css/ietf.scss b/ietf/static/css/ietf.scss index 6695c57b13..b8c701eae1 100644 --- a/ietf/static/css/ietf.scss +++ b/ietf/static/css/ietf.scss @@ -46,6 +46,11 @@ $bootstrap-icons-font-src: url("npm:bootstrap-icons/font/fonts/bootstrap-icons.w url("npm:bootstrap-icons/font/fonts/bootstrap-icons.woff") format("woff"); @import "bootstrap-icons/font/bootstrap-icons"; +// Disable contextual alternates (calt) +body { + font-feature-settings: "calt" off; +} + // Leave room for fixed-top navbar... body.navbar-offset { padding-top: 60px; From 2fba35f76c37f9492b6411c2586c8c75c4aef7ee Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 29 Apr 2026 11:56:31 -0300 Subject: [PATCH 15/17] fix: 2xx == success on red precomputer POST (#10783) --- ietf/doc/utils_red.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/utils_red.py b/ietf/doc/utils_red.py index bcda893dca..5c5879d688 100644 --- a/ietf/doc/utils_red.py +++ b/ietf/doc/utils_red.py @@ -23,7 +23,7 @@ def trigger_red_precomputer(rfc_number_list=()): except requests.Timeout as e: log(f"POST request timed out for {url} : {e}") return - if response.status_code != 200: + if response.status_code // 100 != 2: # 2xx status codes are ok log( f"POST request failed for {url} : status_code={response.status_code}" ) From a7bfb91875332bc88294ca6ab32aeeaf770bd3ad Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 29 Apr 2026 19:59:45 -0300 Subject: [PATCH 16/17] feat: write rfc-index and friends to filesystem (#10784) * feat: write rfc-index and friends to filesystem * style: ruff * fix: create subdir if needed * test: test_save_to_filesystem() * test: check that indexes are written to fs * style: ruff ruff --- ietf/sync/rfcindex.py | 42 +++++++++++++--- ietf/sync/tests_rfcindex.py | 96 +++++++++++++++++++++++++++++-------- 2 files changed, 112 insertions(+), 26 deletions(-) diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index d1a0ed432f..be55a6866e 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -1,8 +1,9 @@ # Copyright The IETF Trust 2026, All Rights Reserved import datetime import json +import shutil from collections import defaultdict -from collections.abc import Container +from collections.abc import Container, Iterable from dataclasses import dataclass from itertools import chain from operator import attrgetter, itemgetter @@ -60,6 +61,25 @@ def save_to_red_bucket(filename: str, content: str | bytes): log(f"Saved {bucket_path} in red_bucket storage") +def save_to_filesystem( + filename: str, content: str | bytes, subdirs: Iterable[str] = () +): + """Save contents to the RFC_PATH in the filesystem + + Always saves directly to settings.RFC_PATH/filename. Additionally saves a copy + to settings.RFC_PATH/subdir/filename for each entry in subdirs. Uses shutil.copy2 + to create the copies, which will preserve mtime and other metadata between copies. + """ + rfc_path = Path(settings.RFC_PATH) + dest_path = rfc_path / filename + dest_path.write_bytes( + content if isinstance(content, bytes) else content.encode("utf-8") + ) + for subdir in subdirs: + (rfc_path / subdir).mkdir(parents=False, exist_ok=True) + shutil.copy2(dest_path, rfc_path / subdir / filename) + + @dataclass class UnusableRfcNumber: rfc_number: int @@ -660,7 +680,9 @@ def create_rfc_txt_index(): "rfcs": get_rfc_text_index_entries(), }, ) - save_to_red_bucket("rfc-index.txt", index) + filename = "rfc-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index) def create_rfc_xml_index(): @@ -696,7 +718,9 @@ def create_rfc_xml_index(): xml_declaration=True, pretty_print=4, ) - save_to_red_bucket("rfc-index.xml", pretty_index) + filename = "rfc-index.xml" + save_to_red_bucket(filename, pretty_index) + save_to_filesystem(filename, pretty_index) def create_bcp_txt_index(): @@ -711,7 +735,9 @@ def create_bcp_txt_index(): "bcps": get_bcp_text_index_entries(), }, ) - save_to_red_bucket("bcp-index.txt", index) + filename = "bcp-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["bcp"]) def create_std_txt_index(): @@ -726,7 +752,9 @@ def create_std_txt_index(): "stds": get_std_text_index_entries(), }, ) - save_to_red_bucket("std-index.txt", index) + filename = "std-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["std"]) def create_fyi_txt_index(): @@ -741,7 +769,9 @@ def create_fyi_txt_index(): "fyis": get_fyi_text_index_entries(), }, ) - save_to_red_bucket("fyi-index.txt", index) + filename = "fyi-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["fyi"]) ## DirtyBits management for the RFC index diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index 541ffbb228..74fa9e7616 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2026, All Rights Reserved import json +from pathlib import Path from unittest import mock +from django.conf import settings from django.core.files.base import ContentFile from django.core.files.storage import storages from django.test.utils import override_settings @@ -27,6 +29,7 @@ get_unusable_rfc_numbers, save_to_red_bucket, subseries_text_line, + save_to_filesystem, ) from ietf.utils.test_utils import TestCase @@ -107,12 +110,17 @@ def tearDown(self): super().tearDown() @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_rfc_txt_index(self, mock_save): + def test_create_rfc_txt_index(self, mock_save_blob, mock_save_file): create_rfc_txt_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "rfc-index.txt") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "rfc-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual(mock_save_file.call_args, mock.call("rfc-index.txt", contents)) + self.assertTrue(isinstance(contents, str)) self.assertIn( "123 Not Issued.", @@ -136,12 +144,17 @@ def test_create_rfc_txt_index(self, mock_save): self.assertNotIn("1 April 2021", contents) @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_rfc_xml_index(self, mock_save): + def test_create_rfc_xml_index(self, mock_save_blob, mock_save_file): create_rfc_xml_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "rfc-index.xml") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "rfc-index.xml") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual(mock_save_file.call_args, mock.call("rfc-index.xml", contents)) + self.assertTrue(isinstance(contents, bytes)) ns = "{https://www.rfc-editor.org/rfc-index}" # NOT an f-string index = etree.fromstring(contents) @@ -204,12 +217,20 @@ def test_create_rfc_xml_index(self, mock_save): ) @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_bcp_txt_index(self, mock_save): + def test_create_bcp_txt_index(self, mock_save_blob, mock_save_file): create_bcp_txt_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "bcp-index.txt") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "bcp-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("bcp-index.txt", contents, ["bcp"]), + ) + self.assertTrue(isinstance(contents, str)) # starts from 1 self.assertIn( @@ -255,12 +276,20 @@ def test_create_bcp_txt_index(self, mock_save): ) @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_std_txt_index(self, mock_save): + def test_create_std_txt_index(self, mock_save_blob, mock_save_file): create_std_txt_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "std-index.txt") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "std-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("std-index.txt", contents, ["std"]), + ) + self.assertTrue(isinstance(contents, str)) # starts from 1 self.assertIn( @@ -306,12 +335,20 @@ def test_create_std_txt_index(self, mock_save): ) @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_fyi_txt_index(self, mock_save): + def test_create_fyi_txt_index(self, mock_save_blob, mock_save_file): create_fyi_txt_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "fyi-index.txt") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "fyi-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("fyi-index.txt", contents, ["fyi"]), + ) + self.assertTrue(isinstance(contents, str)) # starts from 1 self.assertIn( @@ -377,6 +414,25 @@ def test_save_to_red_bucket(self): self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0") red_bucket.delete("test") # clean up like a good child + def test_save_to_filesystem(self): + rfc_path = Path(settings.RFC_PATH) + self.assertFalse((rfc_path / "test").exists()) + save_to_filesystem("test", "contents \U0001f600") + self.assertEqual((rfc_path / "test").read_text("utf-8"), "contents \U0001f600") + self.assertFalse((rfc_path / "subdir" / "test").exists()) + + self.assertFalse((rfc_path / "test2").exists()) + self.assertFalse((rfc_path / "subdir" / "test2").exists()) + save_to_filesystem("test", "contents \U0001f600".encode("utf-8"), ["subdir"]) + self.assertEqual((rfc_path / "test").read_text("utf-8"), "contents \U0001f600") + self.assertEqual( + (rfc_path / "subdir" / "test").read_text("utf-8"), "contents \U0001f600" + ) + self.assertEqual( + (rfc_path / "test").stat().st_mtime, + (rfc_path / "subdir" / "test").stat().st_mtime, + ) + def test_get_unusable_rfc_numbers_raises(self): """get_unusable_rfc_numbers should bail on errors""" with self.assertRaises(FileNotFoundError): From dd6334ae586ac634ff3aab7f598316f86ef0e846 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Thu, 30 Apr 2026 14:53:21 +1200 Subject: [PATCH 17/17] feat: Check if any files exists before publishing RFC (#10782) * style: Ruff ruff Good boy! * feat: Check if any files exists before publishing RFC * refactor: Drop temporary dir creation * refactor: Use helper mixins * test: Add test for blobs * chore: Improve comment --- ietf/api/tests_views_rpc.py | 369 ++++++++++++++++++++---------------- ietf/api/views_rpc.py | 96 ++++++---- 2 files changed, 269 insertions(+), 196 deletions(-) diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 180221cffc..c836cdc2c0 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -2,7 +2,6 @@ import datetime from io import StringIO from pathlib import Path -from tempfile import TemporaryDirectory from django.conf import settings from django.core.files.base import ContentFile @@ -13,8 +12,14 @@ import mock from django.utils import timezone +from ietf.api.views_rpc import DestinationHelperMixin from ietf.blobdb.models import Blob -from ietf.doc.factories import IndividualDraftFactory, RfcFactory, WgDraftFactory, WgRfcFactory +from ietf.doc.factories import ( + IndividualDraftFactory, + RfcFactory, + WgDraftFactory, + WgRfcFactory, +) from ietf.doc.models import RelatedDocument, Document from ietf.group.factories import RoleFactory, GroupFactory from ietf.person.factories import PersonFactory @@ -135,6 +140,27 @@ def test_notify_rfc_published(self, mock_task_delay): r = self.client.post(url, data=post_data, format="json") self.assertEqual(r.status_code, 403) + # Put a file in the way. Post should fail because files exists + rfc_path = Path(settings.RFC_PATH) + (rfc_path / "prerelease").mkdir() + file_in_the_way = rfc_path / f"rfc{unused_rfc_number}.txt" + file_in_the_way.touch() + r = self.client.post( + url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} + ) + self.assertEqual(r.status_code, 409) # conflict + file_in_the_way.unlink() + + # Put a blob in the way. Post should fail because replace = False + blob_in_the_way = Blob.objects.create( + bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b"" + ) + r = self.client.post( + url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} + ) + self.assertEqual(r.status_code, 409) # conflict + blob_in_the_way.delete() + r = self.client.post( url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} ) @@ -193,9 +219,7 @@ def test_notify_rfc_published(self, mock_task_delay): mock_args, mock_kwargs = mock_task_delay.call_args self.assertIn("rfc_number_list", mock_kwargs) expected_rfc_number_list = [rfc.rfc_number] - expected_rfc_number_list.extend( - [d.rfc_number for d in updates + obsoletes] - ) + expected_rfc_number_list.extend([d.rfc_number for d in updates + obsoletes]) expected_rfc_number_list = sorted(set(expected_rfc_number_list)) self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) @@ -238,178 +262,173 @@ def _valid_post_data(): for r in updates: rfc.relateddocument_set.create(relationship_id="updates", target=r) assert isinstance(rfc, Document), "WgRfcFactory should generate a Document" - with TemporaryDirectory() as rfc_dir: - settings.RFC_PATH = rfc_dir # affects overridden settings - rfc_path = Path(rfc_dir) - (rfc_path / "prerelease").mkdir() - content = StringIO("XML content\n") - content.name = "myrfc.xml" + rfc_path = Path(settings.RFC_PATH) + (rfc_path / "prerelease").mkdir() + content = StringIO("XML content\n") + content.name = "myrfc.xml" - # no api key - r = self.client.post(url, _valid_post_data(), format="multipart") - self.assertEqual(r.status_code, 403) - self.assertFalse(mock_update_searchindex_task.delay.called) + # no api key + r = self.client.post(url, _valid_post_data(), format="multipart") + self.assertEqual(r.status_code, 403) + self.assertFalse(mock_update_searchindex_task.delay.called) - # invalid RFC - r = self.client.post( - url, - _valid_post_data() | {"rfc": rfc.rfc_number + 10}, - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 400) - self.assertFalse(mock_update_searchindex_task.delay.called) + # invalid RFC + r = self.client.post( + url, + _valid_post_data() | {"rfc": rfc.rfc_number + 10}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) - # empty files - r = self.client.post( - url, - _valid_post_data() | { - "contents": [ - ContentFile(b"", "myfile.xml"), - ContentFile(b"", "myfile.txt"), - ContentFile(b"", "myfile.html"), - ContentFile(b"", "myfile.pdf"), - ContentFile(b"", "myfile.json"), - ContentFile(b"", "myfile.notprepped.xml"), - ] - }, - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 400) - self.assertFalse(mock_update_searchindex_task.delay.called) + # empty files + r = self.client.post( + url, + _valid_post_data() + | { + "contents": [ + ContentFile(b"", "myfile.xml"), + ContentFile(b"", "myfile.txt"), + ContentFile(b"", "myfile.html"), + ContentFile(b"", "myfile.pdf"), + ContentFile(b"", "myfile.json"), + ContentFile(b"", "myfile.notprepped.xml"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) - # bad file type - r = self.client.post( - url, - _valid_post_data() | { - "contents": [ - ContentFile(b"Some content", "myfile.jpg"), - ] - }, - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 400) - self.assertFalse(mock_update_searchindex_task.delay.called) + # bad file type + r = self.client.post( + url, + _valid_post_data() + | { + "contents": [ + ContentFile(b"Some content", "myfile.jpg"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) - # Put a file in the way. Post should fail because replace = False - file_in_the_way = (rfc_path / f"{rfc.name}.txt") - file_in_the_way.touch() - r = self.client.post( - url, - _valid_post_data(), - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 409) # conflict - self.assertFalse(mock_update_searchindex_task.delay.called) - file_in_the_way.unlink() - - # Put a blob in the way. Post should fail because replace = False - blob_in_the_way = Blob.objects.create( - bucket="rfc", name=f"txt/{rfc.name}.txt", content=b"" - ) - r = self.client.post( - url, - _valid_post_data(), - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 409) # conflict - self.assertFalse(mock_update_searchindex_task.delay.called) - blob_in_the_way.delete() + # Put a file in the way. Post should fail because replace = False + file_in_the_way = rfc_path / f"{rfc.name}.txt" + file_in_the_way.touch() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) + file_in_the_way.unlink() - # valid post - mock_trigger_red_task.delay.reset_mock() - r = self.client.post( - url, - _valid_post_data(), - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 200) - self.assertEqual( - mock_update_searchindex_task.delay.call_args, - mock.call(rfc.rfc_number), - ) - for extension in ["xml", "txt", "html", "pdf", "json"]: - filename = f"{rfc.name}.{extension}" - self.assertEqual( - (rfc_path / filename) - .read_text(), - f"This is .{extension}", - f"{extension} file should contain the expected content", - ) - self.assertEqual( - bytes( - Blob.objects.get( - bucket="rfc", name=f"{extension}/{filename}" - ).content - ), - f"This is .{extension}".encode("utf-8"), - f"{extension} blob should contain the expected content", - ) - # special case for notprepped - notprepped_fn = f"{rfc.name}.notprepped.xml" + # Put a blob in the way. Post should fail because replace = False + blob_in_the_way = Blob.objects.create( + bucket="rfc", name=f"txt/{rfc.name}.txt", content=b"" + ) + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) + blob_in_the_way.delete() + + # valid post + mock_trigger_red_task.delay.reset_mock() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) + for extension in ["xml", "txt", "html", "pdf", "json"]: + filename = f"{rfc.name}.{extension}" self.assertEqual( - ( - rfc_path / "prerelease" / notprepped_fn - ).read_text(), - "This is .notprepped.xml", - ".notprepped.xml file should contain the expected content", + (rfc_path / filename).read_text(), + f"This is .{extension}", + f"{extension} file should contain the expected content", ) self.assertEqual( bytes( Blob.objects.get( - bucket="rfc", name=f"notprepped/{notprepped_fn}" + bucket="rfc", name=f"{extension}/{filename}" ).content ), - b"This is .notprepped.xml", - ".notprepped.xml blob should contain the expected content", - ) - # Confirm that the red precomputer was triggered correctly - self.assertTrue(mock_trigger_red_task.delay.called) - _, mock_kwargs = mock_trigger_red_task.delay.call_args - self.assertIn("rfc_number_list", mock_kwargs) - expected_rfc_number_list = [rfc.rfc_number] - expected_rfc_number_list.extend( - [d.rfc_number for d in updates + obsoletes] + f"This is .{extension}".encode("utf-8"), + f"{extension} blob should contain the expected content", ) - expected_rfc_number_list = sorted(set(expected_rfc_number_list)) - self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) - # Confirm that the search index update task was called correctly - self.assertTrue(mock_update_searchindex_task.delay.called) - # Confirm reference relations rebuild task was called correctly - self.assertTrue(mock_rebuild_relations.delay.called) - _, mock_kwargs = mock_rebuild_relations.delay.call_args - self.assertIn("doc_names", mock_kwargs) - self.assertEqual(mock_kwargs["doc_names"], [rfc.name]) + # special case for notprepped + notprepped_fn = f"{rfc.name}.notprepped.xml" + self.assertEqual( + (rfc_path / "prerelease" / notprepped_fn).read_text(), + "This is .notprepped.xml", + ".notprepped.xml file should contain the expected content", + ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"notprepped/{notprepped_fn}" + ).content + ), + b"This is .notprepped.xml", + ".notprepped.xml blob should contain the expected content", + ) + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_rfc_number_list = [rfc.rfc_number] + expected_rfc_number_list.extend([d.rfc_number for d in updates + obsoletes]) + expected_rfc_number_list = sorted(set(expected_rfc_number_list)) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + # Confirm reference relations rebuild task was called correctly + self.assertTrue(mock_rebuild_relations.delay.called) + _, mock_kwargs = mock_rebuild_relations.delay.call_args + self.assertIn("doc_names", mock_kwargs) + self.assertEqual(mock_kwargs["doc_names"], [rfc.name]) - # re-post with replace = False should now fail - mock_update_searchindex_task.reset_mock() - r = self.client.post( - url, - _valid_post_data(), - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 409) # conflict - self.assertFalse(mock_update_searchindex_task.delay.called) + # re-post with replace = False should now fail + mock_update_searchindex_task.reset_mock() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) - # re-post with replace = True should succeed - r = self.client.post( - url, - _valid_post_data() | {"replace": True}, - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 200) - self.assertTrue(mock_update_searchindex_task.delay.called) - self.assertEqual( - mock_update_searchindex_task.delay.call_args, - mock.call(rfc.rfc_number), - ) + # re-post with replace = True should succeed + r = self.client.post( + url, + _valid_post_data() | {"replace": True}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) def test_refresh_rfc_index(self): @@ -430,3 +449,31 @@ def test_refresh_rfc_index(self): response = self.client.post(url, headers={"X-Api-Key": "valid-token"}) self.assertEqual(response.status_code, 202) self.assertTrue(rfcindex_is_dirty()) + + def test_destination_helper_mixin_fs_destination(self): + file_list = [f"rfc31337.{ext}" for ext in ["txt", "xml", "pdf", "html"]] + for filename in file_list: + self.assertEqual( + DestinationHelperMixin().fs_destination(filename), + Path(f"{settings.RFC_PATH}") / filename, + ) + # noteprepped xml + filename = "rfc31337.notprepped.xml" + self.assertEqual( + DestinationHelperMixin().fs_destination(filename), + Path(f"{settings.RFC_PATH}/prerelease") / filename, + ) + + def test_destination_helper_mixin_blob_destination(self): + file_list = {ext: f"rfc31337.{ext}" for ext in ["txt", "xml", "pdf", "html"]} + for file_type, filename in file_list.items(): + self.assertEqual( + DestinationHelperMixin().blob_destination(filename), + f"{file_type}/{filename}", + ) + # noteprepped xml + filename = "rfc31337.notprepped.xml" + self.assertEqual( + DestinationHelperMixin().blob_destination(filename), + f"notprepped/{filename}", + ) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 6bc45fe3da..e9c17b8a12 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -368,7 +368,38 @@ def get_queryset(self): ) -class RfcPubNotificationView(APIView): +class DestinationHelperMixin: + def fs_destination(self, filename: str | Path) -> Path: + """Destination for an uploaded RFC file in the filesystem + + Strips any path components in filename and returns an absolute Path. + """ + rfc_path = Path(settings.RFC_PATH) + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + return rfc_path / "prerelease" / filename.name + return rfc_path / filename.name + + def blob_destination(self, filename: str | Path) -> str: + """Destination name for an uploaded RFC file in the blob store + + Strips any path components in filename and returns an absolute Path. + """ + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + file_type = "notprepped" + elif extension[0] == ".": + file_type = extension[1:] + else: + raise serializers.ValidationError( + f"Extension does not begin with '.'!? ({filename})", + ) + return f"{file_type}/{filename.name}" + + +class RfcPubNotificationView(DestinationHelperMixin, APIView): api_key_endpoint = "ietf.api.views_rpc" @extend_schema( @@ -380,6 +411,30 @@ class RfcPubNotificationView(APIView): def post(self, request): serializer = RfcPubSerializer(data=request.data) serializer.is_valid(raise_exception=True) + # Check blobstore & filesystem for conflicts + rfc_number = serializer.validated_data["rfc_number"] + dest_stem = f"rfc{rfc_number}" + blob_kind = "rfc" + possible_rfc_files = [ + self.fs_destination(dest_stem + ext) + for ext in RfcFileSerializer.allowed_extensions + ] + possible_rfc_blobs = [ + self.blob_destination(dest_stem + ext) + for ext in RfcFileSerializer.allowed_extensions + ] + for possible_existing_file in possible_rfc_files: + if possible_existing_file.exists(): + raise Conflict( + "File(s) already exist for this RFC", + code="files-exist", + ) + for possible_existing_blob in possible_rfc_blobs: + if exists_in_storage(kind=blob_kind, name=possible_existing_blob): + raise Conflict( + "Blob(s) already exist for this RFC", + code="blobs-exist", + ) # Create RFC try: rfc = serializer.save() @@ -404,39 +459,10 @@ def post(self, request): return Response(NotificationAckSerializer().data) -class RfcPubFilesView(APIView): +class RfcPubFilesView(DestinationHelperMixin, APIView): api_key_endpoint = "ietf.api.views_rpc" parser_classes = [parsers.MultiPartParser] - def _fs_destination(self, filename: str | Path) -> Path: - """Destination for an uploaded RFC file in the filesystem - - Strips any path components in filename and returns an absolute Path. - """ - rfc_path = Path(settings.RFC_PATH) - filename = Path(filename) # could potentially have directory components - extension = "".join(filename.suffixes) - if extension == ".notprepped.xml": - return rfc_path / "prerelease" / filename.name - return rfc_path / filename.name - - def _blob_destination(self, filename: str | Path) -> str: - """Destination name for an uploaded RFC file in the blob store - - Strips any path components in filename and returns an absolute Path. - """ - filename = Path(filename) # could potentially have directory components - extension = "".join(filename.suffixes) - if extension == ".notprepped.xml": - file_type = "notprepped" - elif extension[0] == ".": - file_type = extension[1:] - else: - raise serializers.ValidationError( - f"Extension does not begin with '.'!? ({filename})", - ) - return f"{file_type}/{filename.name}" - @extend_schema( operation_id="upload_rfc_files", summary="Upload files for a published RFC", @@ -459,11 +485,11 @@ def post(self, request): # List of files that might exist for an RFC possible_rfc_files = [ - self._fs_destination(dest_stem + ext) + self.fs_destination(dest_stem + ext) for ext in serializer.allowed_extensions ] possible_rfc_blobs = [ - self._blob_destination(dest_stem + ext) + self.blob_destination(dest_stem + ext) for ext in serializer.allowed_extensions ] if not replace: @@ -508,13 +534,13 @@ def post(self, request): with ftm.open("rb") as f: store_file( kind=blob_kind, - name=self._blob_destination(ftm), + name=self.blob_destination(ftm), file=f, doc_name=rfc.name, doc_rev=rfc.rev, # expect blank, but match whatever it is mtime=mtime, ) - destination = self._fs_destination(ftm) + destination = self.fs_destination(ftm) if ( settings.SERVER_MODE != "production" and not destination.parent.exists()