From d9196b53cfa5d0034ddb9aae7d4e40f91404a3b5 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Thu, 30 Apr 2026 02:56:00 +1200 Subject: [PATCH 1/6] fix: Disable calt font feature (#10779) --- ietf/static/css/ietf.scss | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ietf/static/css/ietf.scss b/ietf/static/css/ietf.scss index 6695c57b13..b8c701eae1 100644 --- a/ietf/static/css/ietf.scss +++ b/ietf/static/css/ietf.scss @@ -46,6 +46,11 @@ $bootstrap-icons-font-src: url("npm:bootstrap-icons/font/fonts/bootstrap-icons.w url("npm:bootstrap-icons/font/fonts/bootstrap-icons.woff") format("woff"); @import "bootstrap-icons/font/bootstrap-icons"; +// Disable contextual alternates (calt) +body { + font-feature-settings: "calt" off; +} + // Leave room for fixed-top navbar... body.navbar-offset { padding-top: 60px; From 2fba35f76c37f9492b6411c2586c8c75c4aef7ee Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 29 Apr 2026 11:56:31 -0300 Subject: [PATCH 2/6] fix: 2xx == success on red precomputer POST (#10783) --- ietf/doc/utils_red.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/utils_red.py b/ietf/doc/utils_red.py index bcda893dca..5c5879d688 100644 --- a/ietf/doc/utils_red.py +++ b/ietf/doc/utils_red.py @@ -23,7 +23,7 @@ def trigger_red_precomputer(rfc_number_list=()): except requests.Timeout as e: log(f"POST request timed out for {url} : {e}") return - if response.status_code != 200: + if response.status_code // 100 != 2: # 2xx status codes are ok log( f"POST request failed for {url} : status_code={response.status_code}" ) From a7bfb91875332bc88294ca6ab32aeeaf770bd3ad Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 29 Apr 2026 19:59:45 -0300 Subject: [PATCH 3/6] feat: write rfc-index and friends to filesystem (#10784) * feat: write rfc-index and friends to filesystem * style: ruff * fix: create subdir if needed * test: test_save_to_filesystem() * test: check that indexes are written to fs * style: ruff ruff --- ietf/sync/rfcindex.py | 42 +++++++++++++--- ietf/sync/tests_rfcindex.py | 96 +++++++++++++++++++++++++++++-------- 2 files changed, 112 insertions(+), 26 deletions(-) diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index d1a0ed432f..be55a6866e 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -1,8 +1,9 @@ # Copyright The IETF Trust 2026, All Rights Reserved import datetime import json +import shutil from collections import defaultdict -from collections.abc import Container +from collections.abc import Container, Iterable from dataclasses import dataclass from itertools import chain from operator import attrgetter, itemgetter @@ -60,6 +61,25 @@ def save_to_red_bucket(filename: str, content: str | bytes): log(f"Saved {bucket_path} in red_bucket storage") +def save_to_filesystem( + filename: str, content: str | bytes, subdirs: Iterable[str] = () +): + """Save contents to the RFC_PATH in the filesystem + + Always saves directly to settings.RFC_PATH/filename. Additionally saves a copy + to settings.RFC_PATH/subdir/filename for each entry in subdirs. Uses shutil.copy2 + to create the copies, which will preserve mtime and other metadata between copies. + """ + rfc_path = Path(settings.RFC_PATH) + dest_path = rfc_path / filename + dest_path.write_bytes( + content if isinstance(content, bytes) else content.encode("utf-8") + ) + for subdir in subdirs: + (rfc_path / subdir).mkdir(parents=False, exist_ok=True) + shutil.copy2(dest_path, rfc_path / subdir / filename) + + @dataclass class UnusableRfcNumber: rfc_number: int @@ -660,7 +680,9 @@ def create_rfc_txt_index(): "rfcs": get_rfc_text_index_entries(), }, ) - save_to_red_bucket("rfc-index.txt", index) + filename = "rfc-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index) def create_rfc_xml_index(): @@ -696,7 +718,9 @@ def create_rfc_xml_index(): xml_declaration=True, pretty_print=4, ) - save_to_red_bucket("rfc-index.xml", pretty_index) + filename = "rfc-index.xml" + save_to_red_bucket(filename, pretty_index) + save_to_filesystem(filename, pretty_index) def create_bcp_txt_index(): @@ -711,7 +735,9 @@ def create_bcp_txt_index(): "bcps": get_bcp_text_index_entries(), }, ) - save_to_red_bucket("bcp-index.txt", index) + filename = "bcp-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["bcp"]) def create_std_txt_index(): @@ -726,7 +752,9 @@ def create_std_txt_index(): "stds": get_std_text_index_entries(), }, ) - save_to_red_bucket("std-index.txt", index) + filename = "std-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["std"]) def create_fyi_txt_index(): @@ -741,7 +769,9 @@ def create_fyi_txt_index(): "fyis": get_fyi_text_index_entries(), }, ) - save_to_red_bucket("fyi-index.txt", index) + filename = "fyi-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["fyi"]) ## DirtyBits management for the RFC index diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index 541ffbb228..74fa9e7616 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2026, All Rights Reserved import json +from pathlib import Path from unittest import mock +from django.conf import settings from django.core.files.base import ContentFile from django.core.files.storage import storages from django.test.utils import override_settings @@ -27,6 +29,7 @@ get_unusable_rfc_numbers, save_to_red_bucket, subseries_text_line, + save_to_filesystem, ) from ietf.utils.test_utils import TestCase @@ -107,12 +110,17 @@ def tearDown(self): super().tearDown() @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_rfc_txt_index(self, mock_save): + def test_create_rfc_txt_index(self, mock_save_blob, mock_save_file): create_rfc_txt_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "rfc-index.txt") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "rfc-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual(mock_save_file.call_args, mock.call("rfc-index.txt", contents)) + self.assertTrue(isinstance(contents, str)) self.assertIn( "123 Not Issued.", @@ -136,12 +144,17 @@ def test_create_rfc_txt_index(self, mock_save): self.assertNotIn("1 April 2021", contents) @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_rfc_xml_index(self, mock_save): + def test_create_rfc_xml_index(self, mock_save_blob, mock_save_file): create_rfc_xml_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "rfc-index.xml") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "rfc-index.xml") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual(mock_save_file.call_args, mock.call("rfc-index.xml", contents)) + self.assertTrue(isinstance(contents, bytes)) ns = "{https://www.rfc-editor.org/rfc-index}" # NOT an f-string index = etree.fromstring(contents) @@ -204,12 +217,20 @@ def test_create_rfc_xml_index(self, mock_save): ) @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_bcp_txt_index(self, mock_save): + def test_create_bcp_txt_index(self, mock_save_blob, mock_save_file): create_bcp_txt_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "bcp-index.txt") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "bcp-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("bcp-index.txt", contents, ["bcp"]), + ) + self.assertTrue(isinstance(contents, str)) # starts from 1 self.assertIn( @@ -255,12 +276,20 @@ def test_create_bcp_txt_index(self, mock_save): ) @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_std_txt_index(self, mock_save): + def test_create_std_txt_index(self, mock_save_blob, mock_save_file): create_std_txt_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "std-index.txt") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "std-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("std-index.txt", contents, ["std"]), + ) + self.assertTrue(isinstance(contents, str)) # starts from 1 self.assertIn( @@ -306,12 +335,20 @@ def test_create_std_txt_index(self, mock_save): ) @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_fyi_txt_index(self, mock_save): + def test_create_fyi_txt_index(self, mock_save_blob, mock_save_file): create_fyi_txt_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "fyi-index.txt") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "fyi-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("fyi-index.txt", contents, ["fyi"]), + ) + self.assertTrue(isinstance(contents, str)) # starts from 1 self.assertIn( @@ -377,6 +414,25 @@ def test_save_to_red_bucket(self): self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0") red_bucket.delete("test") # clean up like a good child + def test_save_to_filesystem(self): + rfc_path = Path(settings.RFC_PATH) + self.assertFalse((rfc_path / "test").exists()) + save_to_filesystem("test", "contents \U0001f600") + self.assertEqual((rfc_path / "test").read_text("utf-8"), "contents \U0001f600") + self.assertFalse((rfc_path / "subdir" / "test").exists()) + + self.assertFalse((rfc_path / "test2").exists()) + self.assertFalse((rfc_path / "subdir" / "test2").exists()) + save_to_filesystem("test", "contents \U0001f600".encode("utf-8"), ["subdir"]) + self.assertEqual((rfc_path / "test").read_text("utf-8"), "contents \U0001f600") + self.assertEqual( + (rfc_path / "subdir" / "test").read_text("utf-8"), "contents \U0001f600" + ) + self.assertEqual( + (rfc_path / "test").stat().st_mtime, + (rfc_path / "subdir" / "test").stat().st_mtime, + ) + def test_get_unusable_rfc_numbers_raises(self): """get_unusable_rfc_numbers should bail on errors""" with self.assertRaises(FileNotFoundError): From dd6334ae586ac634ff3aab7f598316f86ef0e846 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Thu, 30 Apr 2026 14:53:21 +1200 Subject: [PATCH 4/6] feat: Check if any files exists before publishing RFC (#10782) * style: Ruff ruff Good boy! * feat: Check if any files exists before publishing RFC * refactor: Drop temporary dir creation * refactor: Use helper mixins * test: Add test for blobs * chore: Improve comment --- ietf/api/tests_views_rpc.py | 369 ++++++++++++++++++++---------------- ietf/api/views_rpc.py | 96 ++++++---- 2 files changed, 269 insertions(+), 196 deletions(-) diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 180221cffc..c836cdc2c0 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -2,7 +2,6 @@ import datetime from io import StringIO from pathlib import Path -from tempfile import TemporaryDirectory from django.conf import settings from django.core.files.base import ContentFile @@ -13,8 +12,14 @@ import mock from django.utils import timezone +from ietf.api.views_rpc import DestinationHelperMixin from ietf.blobdb.models import Blob -from ietf.doc.factories import IndividualDraftFactory, RfcFactory, WgDraftFactory, WgRfcFactory +from ietf.doc.factories import ( + IndividualDraftFactory, + RfcFactory, + WgDraftFactory, + WgRfcFactory, +) from ietf.doc.models import RelatedDocument, Document from ietf.group.factories import RoleFactory, GroupFactory from ietf.person.factories import PersonFactory @@ -135,6 +140,27 @@ def test_notify_rfc_published(self, mock_task_delay): r = self.client.post(url, data=post_data, format="json") self.assertEqual(r.status_code, 403) + # Put a file in the way. Post should fail because files exists + rfc_path = Path(settings.RFC_PATH) + (rfc_path / "prerelease").mkdir() + file_in_the_way = rfc_path / f"rfc{unused_rfc_number}.txt" + file_in_the_way.touch() + r = self.client.post( + url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} + ) + self.assertEqual(r.status_code, 409) # conflict + file_in_the_way.unlink() + + # Put a blob in the way. Post should fail because replace = False + blob_in_the_way = Blob.objects.create( + bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b"" + ) + r = self.client.post( + url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} + ) + self.assertEqual(r.status_code, 409) # conflict + blob_in_the_way.delete() + r = self.client.post( url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} ) @@ -193,9 +219,7 @@ def test_notify_rfc_published(self, mock_task_delay): mock_args, mock_kwargs = mock_task_delay.call_args self.assertIn("rfc_number_list", mock_kwargs) expected_rfc_number_list = [rfc.rfc_number] - expected_rfc_number_list.extend( - [d.rfc_number for d in updates + obsoletes] - ) + expected_rfc_number_list.extend([d.rfc_number for d in updates + obsoletes]) expected_rfc_number_list = sorted(set(expected_rfc_number_list)) self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) @@ -238,178 +262,173 @@ def _valid_post_data(): for r in updates: rfc.relateddocument_set.create(relationship_id="updates", target=r) assert isinstance(rfc, Document), "WgRfcFactory should generate a Document" - with TemporaryDirectory() as rfc_dir: - settings.RFC_PATH = rfc_dir # affects overridden settings - rfc_path = Path(rfc_dir) - (rfc_path / "prerelease").mkdir() - content = StringIO("XML content\n") - content.name = "myrfc.xml" + rfc_path = Path(settings.RFC_PATH) + (rfc_path / "prerelease").mkdir() + content = StringIO("XML content\n") + content.name = "myrfc.xml" - # no api key - r = self.client.post(url, _valid_post_data(), format="multipart") - self.assertEqual(r.status_code, 403) - self.assertFalse(mock_update_searchindex_task.delay.called) + # no api key + r = self.client.post(url, _valid_post_data(), format="multipart") + self.assertEqual(r.status_code, 403) + self.assertFalse(mock_update_searchindex_task.delay.called) - # invalid RFC - r = self.client.post( - url, - _valid_post_data() | {"rfc": rfc.rfc_number + 10}, - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 400) - self.assertFalse(mock_update_searchindex_task.delay.called) + # invalid RFC + r = self.client.post( + url, + _valid_post_data() | {"rfc": rfc.rfc_number + 10}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) - # empty files - r = self.client.post( - url, - _valid_post_data() | { - "contents": [ - ContentFile(b"", "myfile.xml"), - ContentFile(b"", "myfile.txt"), - ContentFile(b"", "myfile.html"), - ContentFile(b"", "myfile.pdf"), - ContentFile(b"", "myfile.json"), - ContentFile(b"", "myfile.notprepped.xml"), - ] - }, - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 400) - self.assertFalse(mock_update_searchindex_task.delay.called) + # empty files + r = self.client.post( + url, + _valid_post_data() + | { + "contents": [ + ContentFile(b"", "myfile.xml"), + ContentFile(b"", "myfile.txt"), + ContentFile(b"", "myfile.html"), + ContentFile(b"", "myfile.pdf"), + ContentFile(b"", "myfile.json"), + ContentFile(b"", "myfile.notprepped.xml"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) - # bad file type - r = self.client.post( - url, - _valid_post_data() | { - "contents": [ - ContentFile(b"Some content", "myfile.jpg"), - ] - }, - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 400) - self.assertFalse(mock_update_searchindex_task.delay.called) + # bad file type + r = self.client.post( + url, + _valid_post_data() + | { + "contents": [ + ContentFile(b"Some content", "myfile.jpg"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) - # Put a file in the way. Post should fail because replace = False - file_in_the_way = (rfc_path / f"{rfc.name}.txt") - file_in_the_way.touch() - r = self.client.post( - url, - _valid_post_data(), - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 409) # conflict - self.assertFalse(mock_update_searchindex_task.delay.called) - file_in_the_way.unlink() - - # Put a blob in the way. Post should fail because replace = False - blob_in_the_way = Blob.objects.create( - bucket="rfc", name=f"txt/{rfc.name}.txt", content=b"" - ) - r = self.client.post( - url, - _valid_post_data(), - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 409) # conflict - self.assertFalse(mock_update_searchindex_task.delay.called) - blob_in_the_way.delete() + # Put a file in the way. Post should fail because replace = False + file_in_the_way = rfc_path / f"{rfc.name}.txt" + file_in_the_way.touch() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) + file_in_the_way.unlink() - # valid post - mock_trigger_red_task.delay.reset_mock() - r = self.client.post( - url, - _valid_post_data(), - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 200) - self.assertEqual( - mock_update_searchindex_task.delay.call_args, - mock.call(rfc.rfc_number), - ) - for extension in ["xml", "txt", "html", "pdf", "json"]: - filename = f"{rfc.name}.{extension}" - self.assertEqual( - (rfc_path / filename) - .read_text(), - f"This is .{extension}", - f"{extension} file should contain the expected content", - ) - self.assertEqual( - bytes( - Blob.objects.get( - bucket="rfc", name=f"{extension}/{filename}" - ).content - ), - f"This is .{extension}".encode("utf-8"), - f"{extension} blob should contain the expected content", - ) - # special case for notprepped - notprepped_fn = f"{rfc.name}.notprepped.xml" + # Put a blob in the way. Post should fail because replace = False + blob_in_the_way = Blob.objects.create( + bucket="rfc", name=f"txt/{rfc.name}.txt", content=b"" + ) + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) + blob_in_the_way.delete() + + # valid post + mock_trigger_red_task.delay.reset_mock() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) + for extension in ["xml", "txt", "html", "pdf", "json"]: + filename = f"{rfc.name}.{extension}" self.assertEqual( - ( - rfc_path / "prerelease" / notprepped_fn - ).read_text(), - "This is .notprepped.xml", - ".notprepped.xml file should contain the expected content", + (rfc_path / filename).read_text(), + f"This is .{extension}", + f"{extension} file should contain the expected content", ) self.assertEqual( bytes( Blob.objects.get( - bucket="rfc", name=f"notprepped/{notprepped_fn}" + bucket="rfc", name=f"{extension}/{filename}" ).content ), - b"This is .notprepped.xml", - ".notprepped.xml blob should contain the expected content", - ) - # Confirm that the red precomputer was triggered correctly - self.assertTrue(mock_trigger_red_task.delay.called) - _, mock_kwargs = mock_trigger_red_task.delay.call_args - self.assertIn("rfc_number_list", mock_kwargs) - expected_rfc_number_list = [rfc.rfc_number] - expected_rfc_number_list.extend( - [d.rfc_number for d in updates + obsoletes] + f"This is .{extension}".encode("utf-8"), + f"{extension} blob should contain the expected content", ) - expected_rfc_number_list = sorted(set(expected_rfc_number_list)) - self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) - # Confirm that the search index update task was called correctly - self.assertTrue(mock_update_searchindex_task.delay.called) - # Confirm reference relations rebuild task was called correctly - self.assertTrue(mock_rebuild_relations.delay.called) - _, mock_kwargs = mock_rebuild_relations.delay.call_args - self.assertIn("doc_names", mock_kwargs) - self.assertEqual(mock_kwargs["doc_names"], [rfc.name]) + # special case for notprepped + notprepped_fn = f"{rfc.name}.notprepped.xml" + self.assertEqual( + (rfc_path / "prerelease" / notprepped_fn).read_text(), + "This is .notprepped.xml", + ".notprepped.xml file should contain the expected content", + ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"notprepped/{notprepped_fn}" + ).content + ), + b"This is .notprepped.xml", + ".notprepped.xml blob should contain the expected content", + ) + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_rfc_number_list = [rfc.rfc_number] + expected_rfc_number_list.extend([d.rfc_number for d in updates + obsoletes]) + expected_rfc_number_list = sorted(set(expected_rfc_number_list)) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + # Confirm reference relations rebuild task was called correctly + self.assertTrue(mock_rebuild_relations.delay.called) + _, mock_kwargs = mock_rebuild_relations.delay.call_args + self.assertIn("doc_names", mock_kwargs) + self.assertEqual(mock_kwargs["doc_names"], [rfc.name]) - # re-post with replace = False should now fail - mock_update_searchindex_task.reset_mock() - r = self.client.post( - url, - _valid_post_data(), - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 409) # conflict - self.assertFalse(mock_update_searchindex_task.delay.called) + # re-post with replace = False should now fail + mock_update_searchindex_task.reset_mock() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) - # re-post with replace = True should succeed - r = self.client.post( - url, - _valid_post_data() | {"replace": True}, - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) - self.assertEqual(r.status_code, 200) - self.assertTrue(mock_update_searchindex_task.delay.called) - self.assertEqual( - mock_update_searchindex_task.delay.call_args, - mock.call(rfc.rfc_number), - ) + # re-post with replace = True should succeed + r = self.client.post( + url, + _valid_post_data() | {"replace": True}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) def test_refresh_rfc_index(self): @@ -430,3 +449,31 @@ def test_refresh_rfc_index(self): response = self.client.post(url, headers={"X-Api-Key": "valid-token"}) self.assertEqual(response.status_code, 202) self.assertTrue(rfcindex_is_dirty()) + + def test_destination_helper_mixin_fs_destination(self): + file_list = [f"rfc31337.{ext}" for ext in ["txt", "xml", "pdf", "html"]] + for filename in file_list: + self.assertEqual( + DestinationHelperMixin().fs_destination(filename), + Path(f"{settings.RFC_PATH}") / filename, + ) + # noteprepped xml + filename = "rfc31337.notprepped.xml" + self.assertEqual( + DestinationHelperMixin().fs_destination(filename), + Path(f"{settings.RFC_PATH}/prerelease") / filename, + ) + + def test_destination_helper_mixin_blob_destination(self): + file_list = {ext: f"rfc31337.{ext}" for ext in ["txt", "xml", "pdf", "html"]} + for file_type, filename in file_list.items(): + self.assertEqual( + DestinationHelperMixin().blob_destination(filename), + f"{file_type}/{filename}", + ) + # noteprepped xml + filename = "rfc31337.notprepped.xml" + self.assertEqual( + DestinationHelperMixin().blob_destination(filename), + f"notprepped/{filename}", + ) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 6bc45fe3da..e9c17b8a12 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -368,7 +368,38 @@ def get_queryset(self): ) -class RfcPubNotificationView(APIView): +class DestinationHelperMixin: + def fs_destination(self, filename: str | Path) -> Path: + """Destination for an uploaded RFC file in the filesystem + + Strips any path components in filename and returns an absolute Path. + """ + rfc_path = Path(settings.RFC_PATH) + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + return rfc_path / "prerelease" / filename.name + return rfc_path / filename.name + + def blob_destination(self, filename: str | Path) -> str: + """Destination name for an uploaded RFC file in the blob store + + Strips any path components in filename and returns an absolute Path. + """ + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + file_type = "notprepped" + elif extension[0] == ".": + file_type = extension[1:] + else: + raise serializers.ValidationError( + f"Extension does not begin with '.'!? ({filename})", + ) + return f"{file_type}/{filename.name}" + + +class RfcPubNotificationView(DestinationHelperMixin, APIView): api_key_endpoint = "ietf.api.views_rpc" @extend_schema( @@ -380,6 +411,30 @@ class RfcPubNotificationView(APIView): def post(self, request): serializer = RfcPubSerializer(data=request.data) serializer.is_valid(raise_exception=True) + # Check blobstore & filesystem for conflicts + rfc_number = serializer.validated_data["rfc_number"] + dest_stem = f"rfc{rfc_number}" + blob_kind = "rfc" + possible_rfc_files = [ + self.fs_destination(dest_stem + ext) + for ext in RfcFileSerializer.allowed_extensions + ] + possible_rfc_blobs = [ + self.blob_destination(dest_stem + ext) + for ext in RfcFileSerializer.allowed_extensions + ] + for possible_existing_file in possible_rfc_files: + if possible_existing_file.exists(): + raise Conflict( + "File(s) already exist for this RFC", + code="files-exist", + ) + for possible_existing_blob in possible_rfc_blobs: + if exists_in_storage(kind=blob_kind, name=possible_existing_blob): + raise Conflict( + "Blob(s) already exist for this RFC", + code="blobs-exist", + ) # Create RFC try: rfc = serializer.save() @@ -404,39 +459,10 @@ def post(self, request): return Response(NotificationAckSerializer().data) -class RfcPubFilesView(APIView): +class RfcPubFilesView(DestinationHelperMixin, APIView): api_key_endpoint = "ietf.api.views_rpc" parser_classes = [parsers.MultiPartParser] - def _fs_destination(self, filename: str | Path) -> Path: - """Destination for an uploaded RFC file in the filesystem - - Strips any path components in filename and returns an absolute Path. - """ - rfc_path = Path(settings.RFC_PATH) - filename = Path(filename) # could potentially have directory components - extension = "".join(filename.suffixes) - if extension == ".notprepped.xml": - return rfc_path / "prerelease" / filename.name - return rfc_path / filename.name - - def _blob_destination(self, filename: str | Path) -> str: - """Destination name for an uploaded RFC file in the blob store - - Strips any path components in filename and returns an absolute Path. - """ - filename = Path(filename) # could potentially have directory components - extension = "".join(filename.suffixes) - if extension == ".notprepped.xml": - file_type = "notprepped" - elif extension[0] == ".": - file_type = extension[1:] - else: - raise serializers.ValidationError( - f"Extension does not begin with '.'!? ({filename})", - ) - return f"{file_type}/{filename.name}" - @extend_schema( operation_id="upload_rfc_files", summary="Upload files for a published RFC", @@ -459,11 +485,11 @@ def post(self, request): # List of files that might exist for an RFC possible_rfc_files = [ - self._fs_destination(dest_stem + ext) + self.fs_destination(dest_stem + ext) for ext in serializer.allowed_extensions ] possible_rfc_blobs = [ - self._blob_destination(dest_stem + ext) + self.blob_destination(dest_stem + ext) for ext in serializer.allowed_extensions ] if not replace: @@ -508,13 +534,13 @@ def post(self, request): with ftm.open("rb") as f: store_file( kind=blob_kind, - name=self._blob_destination(ftm), + name=self.blob_destination(ftm), file=f, doc_name=rfc.name, doc_rev=rfc.rev, # expect blank, but match whatever it is mtime=mtime, ) - destination = self._fs_destination(ftm) + destination = self.fs_destination(ftm) if ( settings.SERVER_MODE != "production" and not destination.parent.exists() From 7f25c287d83545a9123117e112baf4b585d7c59c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 30 Apr 2026 13:15:47 -0300 Subject: [PATCH 5/6] chore: add DATATRACKER_BROKER_URL config option (#10785) --- k8s/settings_local.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 251f11234f..19d0a1c2f5 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -155,14 +155,18 @@ def _multiline_to_list(s): EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost") EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025")) +_broker_url = os.environ.get("DATATRACKER_BROKER_URL", None) _celery_password = os.environ.get("CELERY_PASSWORD", None) -if _celery_password is None: - raise RuntimeError("CELERY_PASSWORD must be set") -CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( - host=os.environ.get("RABBITMQ_HOSTNAME", "dt-rabbitmq"), - password=_celery_password, - queue=os.environ.get("RABBITMQ_QUEUE", "dt"), -) +if _broker_url is not None: + CELERY_BROKER_URL = _broker_url +elif _celery_password is not None: + CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( + host=os.environ.get("RABBITMQ_HOSTNAME", "dt-rabbitmq"), + password=_celery_password, + queue=os.environ.get("RABBITMQ_QUEUE", "dt"), + ) +else: + raise RuntimeError("DATATRACKER_BROKER_URL or CELERY_PASSWORD must be set") # mailarchive API key _mailing_list_archive_api_key = os.environ.get( From 56009e7dda34e92acdda1b2b3301cefe8b203df2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 30 Apr 2026 14:34:50 -0300 Subject: [PATCH 6/6] ci: increase staging/prod deploy timeouts --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 49a0e5b53b..ff84408187 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -466,7 +466,7 @@ jobs: token: ${{ secrets.GH_INFRA_K8S_TOKEN }} inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' wait-for-completion: true - wait-for-completion-timeout: 10m + wait-for-completion-timeout: 30m wait-for-completion-interval: 30s display-workflow-run-url: false @@ -493,6 +493,6 @@ jobs: token: ${{ secrets.GH_INFRA_K8S_TOKEN }} inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' wait-for-completion: true - wait-for-completion-timeout: 10m + wait-for-completion-timeout: 30m wait-for-completion-interval: 30s display-workflow-run-url: false