diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index d1a0ed432f..be55a6866e 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -1,8 +1,9 @@ # Copyright The IETF Trust 2026, All Rights Reserved import datetime import json +import shutil from collections import defaultdict -from collections.abc import Container +from collections.abc import Container, Iterable from dataclasses import dataclass from itertools import chain from operator import attrgetter, itemgetter @@ -60,6 +61,25 @@ def save_to_red_bucket(filename: str, content: str | bytes): log(f"Saved {bucket_path} in red_bucket storage") +def save_to_filesystem( + filename: str, content: str | bytes, subdirs: Iterable[str] = () +): + """Save contents to the RFC_PATH in the filesystem + + Always saves directly to settings.RFC_PATH/filename. Additionally saves a copy + to settings.RFC_PATH/subdir/filename for each entry in subdirs. Uses shutil.copy2 + to create the copies, which will preserve mtime and other metadata between copies. + """ + rfc_path = Path(settings.RFC_PATH) + dest_path = rfc_path / filename + dest_path.write_bytes( + content if isinstance(content, bytes) else content.encode("utf-8") + ) + for subdir in subdirs: + (rfc_path / subdir).mkdir(parents=False, exist_ok=True) + shutil.copy2(dest_path, rfc_path / subdir / filename) + + @dataclass class UnusableRfcNumber: rfc_number: int @@ -660,7 +680,9 @@ def create_rfc_txt_index(): "rfcs": get_rfc_text_index_entries(), }, ) - save_to_red_bucket("rfc-index.txt", index) + filename = "rfc-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index) def create_rfc_xml_index(): @@ -696,7 +718,9 @@ def create_rfc_xml_index(): xml_declaration=True, pretty_print=4, ) - save_to_red_bucket("rfc-index.xml", pretty_index) + filename = "rfc-index.xml" + save_to_red_bucket(filename, pretty_index) + save_to_filesystem(filename, pretty_index) def create_bcp_txt_index(): @@ -711,7 +735,9 @@ def create_bcp_txt_index(): "bcps": get_bcp_text_index_entries(), }, ) - save_to_red_bucket("bcp-index.txt", index) + filename = "bcp-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["bcp"]) def create_std_txt_index(): @@ -726,7 +752,9 @@ def create_std_txt_index(): "stds": get_std_text_index_entries(), }, ) - save_to_red_bucket("std-index.txt", index) + filename = "std-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["std"]) def create_fyi_txt_index(): @@ -741,7 +769,9 @@ def create_fyi_txt_index(): "fyis": get_fyi_text_index_entries(), }, ) - save_to_red_bucket("fyi-index.txt", index) + filename = "fyi-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["fyi"]) ## DirtyBits management for the RFC index diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index 541ffbb228..74fa9e7616 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2026, All Rights Reserved import json +from pathlib import Path from unittest import mock +from django.conf import settings from django.core.files.base import ContentFile from django.core.files.storage import storages from django.test.utils import override_settings @@ -27,6 +29,7 @@ get_unusable_rfc_numbers, save_to_red_bucket, subseries_text_line, + save_to_filesystem, ) from ietf.utils.test_utils import TestCase @@ -107,12 +110,17 @@ def tearDown(self): super().tearDown() @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_rfc_txt_index(self, mock_save): + def test_create_rfc_txt_index(self, mock_save_blob, mock_save_file): create_rfc_txt_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "rfc-index.txt") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "rfc-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual(mock_save_file.call_args, mock.call("rfc-index.txt", contents)) + self.assertTrue(isinstance(contents, str)) self.assertIn( "123 Not Issued.", @@ -136,12 +144,17 @@ def test_create_rfc_txt_index(self, mock_save): self.assertNotIn("1 April 2021", contents) @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_rfc_xml_index(self, mock_save): + def test_create_rfc_xml_index(self, mock_save_blob, mock_save_file): create_rfc_xml_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "rfc-index.xml") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "rfc-index.xml") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual(mock_save_file.call_args, mock.call("rfc-index.xml", contents)) + self.assertTrue(isinstance(contents, bytes)) ns = "{https://www.rfc-editor.org/rfc-index}" # NOT an f-string index = etree.fromstring(contents) @@ -204,12 +217,20 @@ def test_create_rfc_xml_index(self, mock_save): ) @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_bcp_txt_index(self, mock_save): + def test_create_bcp_txt_index(self, mock_save_blob, mock_save_file): create_bcp_txt_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "bcp-index.txt") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "bcp-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("bcp-index.txt", contents, ["bcp"]), + ) + self.assertTrue(isinstance(contents, str)) # starts from 1 self.assertIn( @@ -255,12 +276,20 @@ def test_create_bcp_txt_index(self, mock_save): ) @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_std_txt_index(self, mock_save): + def test_create_std_txt_index(self, mock_save_blob, mock_save_file): create_std_txt_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "std-index.txt") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "std-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("std-index.txt", contents, ["std"]), + ) + self.assertTrue(isinstance(contents, str)) # starts from 1 self.assertIn( @@ -306,12 +335,20 @@ def test_create_std_txt_index(self, mock_save): ) @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") - def test_create_fyi_txt_index(self, mock_save): + def test_create_fyi_txt_index(self, mock_save_blob, mock_save_file): create_fyi_txt_index() - self.assertEqual(mock_save.call_count, 1) - self.assertEqual(mock_save.call_args[0][0], "fyi-index.txt") - contents = mock_save.call_args[0][1] + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "fyi-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("fyi-index.txt", contents, ["fyi"]), + ) + self.assertTrue(isinstance(contents, str)) # starts from 1 self.assertIn( @@ -377,6 +414,25 @@ def test_save_to_red_bucket(self): self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0") red_bucket.delete("test") # clean up like a good child + def test_save_to_filesystem(self): + rfc_path = Path(settings.RFC_PATH) + self.assertFalse((rfc_path / "test").exists()) + save_to_filesystem("test", "contents \U0001f600") + self.assertEqual((rfc_path / "test").read_text("utf-8"), "contents \U0001f600") + self.assertFalse((rfc_path / "subdir" / "test").exists()) + + self.assertFalse((rfc_path / "test2").exists()) + self.assertFalse((rfc_path / "subdir" / "test2").exists()) + save_to_filesystem("test", "contents \U0001f600".encode("utf-8"), ["subdir"]) + self.assertEqual((rfc_path / "test").read_text("utf-8"), "contents \U0001f600") + self.assertEqual( + (rfc_path / "subdir" / "test").read_text("utf-8"), "contents \U0001f600" + ) + self.assertEqual( + (rfc_path / "test").stat().st_mtime, + (rfc_path / "subdir" / "test").stat().st_mtime, + ) + def test_get_unusable_rfc_numbers_raises(self): """get_unusable_rfc_numbers should bail on errors""" with self.assertRaises(FileNotFoundError):