feat: write rfc-index and friends to filesystem

ietf/sync/rfcindex.py

@@ -1,8 +1,9 @@
 # Copyright The IETF Trust 2026, All Rights Reserved
 import datetime
 import json
+import shutil
 from collections import defaultdict
-from collections.abc import Container
+from collections.abc import Container, Iterable
 from dataclasses import dataclass
 from itertools import chain
 from operator import attrgetter, itemgetter
@@ -60,6 +61,25 @@ def save_to_red_bucket(filename: str, content: str | bytes):
     log(f"Saved {bucket_path} in red_bucket storage")
 
 
+def save_to_filesystem(
+    filename: str, content: str | bytes, subdirs: Iterable[str] = ()
+):
+    """Save contents to the RFC_PATH in the filesystem
+
+    Always saves directly to settings.RFC_PATH/filename. Additionally saves a copy
+    to settings.RFC_PATH/subdir/filename for each entry in subdirs. Uses shutil.copy2
+    to create the copies, which will preserve mtime and other metadata between copies.
+    """
+    rfc_path = Path(settings.RFC_PATH)
+    dest_path = rfc_path / filename
+    dest_path.write_bytes(
+        content if isinstance(content, bytes) else content.encode("utf-8")
+    )
+    for subdir in subdirs:
+        (rfc_path / subdir).mkdir(parents=False, exist_ok=True)
+        shutil.copy2(dest_path, rfc_path / subdir / filename)
+
+
 @dataclass
 class UnusableRfcNumber:
     rfc_number: int
@@ -660,7 +680,9 @@ def create_rfc_txt_index():
             "rfcs": get_rfc_text_index_entries(),
         },
     )
-    save_to_red_bucket("rfc-index.txt", index)
+    filename = "rfc-index.txt"
+    save_to_red_bucket(filename, index)
+    save_to_filesystem(filename, index)
 
 
 def create_rfc_xml_index():
@@ -696,7 +718,9 @@ def create_rfc_xml_index():
         xml_declaration=True,
         pretty_print=4,
     )
-    save_to_red_bucket("rfc-index.xml", pretty_index)
+    filename = "rfc-index.xml"
+    save_to_red_bucket(filename, pretty_index)
+    save_to_filesystem(filename, pretty_index)
 
 
 def create_bcp_txt_index():
@@ -711,7 +735,9 @@ def create_bcp_txt_index():
             "bcps": get_bcp_text_index_entries(),
         },
     )
-    save_to_red_bucket("bcp-index.txt", index)
+    filename = "bcp-index.txt"
+    save_to_red_bucket(filename, index)
+    save_to_filesystem(filename, index, ["bcp"])
 
 
 def create_std_txt_index():
@@ -726,7 +752,9 @@ def create_std_txt_index():
             "stds": get_std_text_index_entries(),
         },
     )
-    save_to_red_bucket("std-index.txt", index)
+    filename = "std-index.txt"
+    save_to_red_bucket(filename, index)
+    save_to_filesystem(filename, index, ["std"])
 
 
 def create_fyi_txt_index():
@@ -741,7 +769,9 @@ def create_fyi_txt_index():
             "fyis": get_fyi_text_index_entries(),
         },
     )
-    save_to_red_bucket("fyi-index.txt", index)
+    filename = "fyi-index.txt"
+    save_to_red_bucket(filename, index)
+    save_to_filesystem(filename, index, ["fyi"])
 
 
 ## DirtyBits management for the RFC index

ietf/sync/tests_rfcindex.py

@@ -1,7 +1,9 @@
 # Copyright The IETF Trust 2026, All Rights Reserved
 import json
+from pathlib import Path
 from unittest import mock
 
+from django.conf import settings
 from django.core.files.base import ContentFile
 from django.core.files.storage import storages
 from django.test.utils import override_settings
@@ -27,6 +29,7 @@
     get_unusable_rfc_numbers,
     save_to_red_bucket,
     subseries_text_line,
+    save_to_filesystem,
 )
 from ietf.utils.test_utils import TestCase
 
@@ -107,12 +110,17 @@ def tearDown(self):
         super().tearDown()
 
     @override_settings(RFCINDEX_INPUT_PATH="input/")
+    @mock.patch("ietf.sync.rfcindex.save_to_filesystem")
     @mock.patch("ietf.sync.rfcindex.save_to_red_bucket")
-    def test_create_rfc_txt_index(self, mock_save):
+    def test_create_rfc_txt_index(self, mock_save_blob, mock_save_file):
         create_rfc_txt_index()
-        self.assertEqual(mock_save.call_count, 1)
-        self.assertEqual(mock_save.call_args[0][0], "rfc-index.txt")
-        contents = mock_save.call_args[0][1]
+        self.assertEqual(mock_save_blob.call_count, 1)
+        self.assertEqual(mock_save_blob.call_args[0][0], "rfc-index.txt")
+        contents = mock_save_blob.call_args[0][1]
+
+        self.assertEqual(mock_save_file.call_count, 1)
+        self.assertEqual(mock_save_file.call_args, mock.call("rfc-index.txt", contents))
+
         self.assertTrue(isinstance(contents, str))
         self.assertIn(
             "123 Not Issued.",
@@ -136,12 +144,17 @@ def test_create_rfc_txt_index(self, mock_save):
         self.assertNotIn("1 April 2021", contents)
 
     @override_settings(RFCINDEX_INPUT_PATH="input/")
+    @mock.patch("ietf.sync.rfcindex.save_to_filesystem")
     @mock.patch("ietf.sync.rfcindex.save_to_red_bucket")
-    def test_create_rfc_xml_index(self, mock_save):
+    def test_create_rfc_xml_index(self, mock_save_blob, mock_save_file):
         create_rfc_xml_index()
-        self.assertEqual(mock_save.call_count, 1)
-        self.assertEqual(mock_save.call_args[0][0], "rfc-index.xml")
-        contents = mock_save.call_args[0][1]
+        self.assertEqual(mock_save_blob.call_count, 1)
+        self.assertEqual(mock_save_blob.call_args[0][0], "rfc-index.xml")
+        contents = mock_save_blob.call_args[0][1]
+
+        self.assertEqual(mock_save_file.call_count, 1)
+        self.assertEqual(mock_save_file.call_args, mock.call("rfc-index.xml", contents))
+
         self.assertTrue(isinstance(contents, bytes))
         ns = "{https://www.rfc-editor.org/rfc-index}"  # NOT an f-string
         index = etree.fromstring(contents)
@@ -204,12 +217,20 @@ def test_create_rfc_xml_index(self, mock_save):
         )
 
     @override_settings(RFCINDEX_INPUT_PATH="input/")
+    @mock.patch("ietf.sync.rfcindex.save_to_filesystem")
     @mock.patch("ietf.sync.rfcindex.save_to_red_bucket")
-    def test_create_bcp_txt_index(self, mock_save):
+    def test_create_bcp_txt_index(self, mock_save_blob, mock_save_file):
         create_bcp_txt_index()
-        self.assertEqual(mock_save.call_count, 1)
-        self.assertEqual(mock_save.call_args[0][0], "bcp-index.txt")
-        contents = mock_save.call_args[0][1]
+        self.assertEqual(mock_save_blob.call_count, 1)
+        self.assertEqual(mock_save_blob.call_args[0][0], "bcp-index.txt")
+        contents = mock_save_blob.call_args[0][1]
+
+        self.assertEqual(mock_save_file.call_count, 1)
+        self.assertEqual(
+            mock_save_file.call_args,
+            mock.call("bcp-index.txt", contents, ["bcp"]),
+        )
+
         self.assertTrue(isinstance(contents, str))
         # starts from 1
         self.assertIn(
@@ -255,12 +276,20 @@ def test_create_bcp_txt_index(self, mock_save):
         )
 
     @override_settings(RFCINDEX_INPUT_PATH="input/")
+    @mock.patch("ietf.sync.rfcindex.save_to_filesystem")
     @mock.patch("ietf.sync.rfcindex.save_to_red_bucket")
-    def test_create_std_txt_index(self, mock_save):
+    def test_create_std_txt_index(self, mock_save_blob, mock_save_file):
         create_std_txt_index()
-        self.assertEqual(mock_save.call_count, 1)
-        self.assertEqual(mock_save.call_args[0][0], "std-index.txt")
-        contents = mock_save.call_args[0][1]
+        self.assertEqual(mock_save_blob.call_count, 1)
+        self.assertEqual(mock_save_blob.call_args[0][0], "std-index.txt")
+        contents = mock_save_blob.call_args[0][1]
+
+        self.assertEqual(mock_save_file.call_count, 1)
+        self.assertEqual(
+            mock_save_file.call_args,
+            mock.call("std-index.txt", contents, ["std"]),
+        )
+
         self.assertTrue(isinstance(contents, str))
         # starts from 1
         self.assertIn(
@@ -306,12 +335,20 @@ def test_create_std_txt_index(self, mock_save):
         )
 
     @override_settings(RFCINDEX_INPUT_PATH="input/")
+    @mock.patch("ietf.sync.rfcindex.save_to_filesystem")
     @mock.patch("ietf.sync.rfcindex.save_to_red_bucket")
-    def test_create_fyi_txt_index(self, mock_save):
+    def test_create_fyi_txt_index(self, mock_save_blob, mock_save_file):
         create_fyi_txt_index()
-        self.assertEqual(mock_save.call_count, 1)
-        self.assertEqual(mock_save.call_args[0][0], "fyi-index.txt")
-        contents = mock_save.call_args[0][1]
+        self.assertEqual(mock_save_blob.call_count, 1)
+        self.assertEqual(mock_save_blob.call_args[0][0], "fyi-index.txt")
+        contents = mock_save_blob.call_args[0][1]
+
+        self.assertEqual(mock_save_file.call_count, 1)
+        self.assertEqual(
+            mock_save_file.call_args,
+            mock.call("fyi-index.txt", contents, ["fyi"]),
+        )
+
         self.assertTrue(isinstance(contents, str))
         # starts from 1
         self.assertIn(
@@ -377,6 +414,25 @@ def test_save_to_red_bucket(self):
             self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0")
         red_bucket.delete("test")  # clean up like a good child
 
+    def test_save_to_filesystem(self):
+        rfc_path = Path(settings.RFC_PATH)
+        self.assertFalse((rfc_path / "test").exists())
+        save_to_filesystem("test", "contents \U0001f600")
+        self.assertEqual((rfc_path / "test").read_text("utf-8"), "contents \U0001f600")
+        self.assertFalse((rfc_path / "subdir" / "test").exists())
+
+        self.assertFalse((rfc_path / "test2").exists())
+        self.assertFalse((rfc_path / "subdir" / "test2").exists())
+        save_to_filesystem("test", "contents \U0001f600".encode("utf-8"), ["subdir"])
+        self.assertEqual((rfc_path / "test").read_text("utf-8"), "contents \U0001f600")
+        self.assertEqual(
+            (rfc_path / "subdir" / "test").read_text("utf-8"), "contents \U0001f600"
+        )
+        self.assertEqual(
+            (rfc_path / "test").stat().st_mtime,
+            (rfc_path / "subdir" / "test").stat().st_mtime,
+        )
+
     def test_get_unusable_rfc_numbers_raises(self):
         """get_unusable_rfc_numbers should bail on errors"""
         with self.assertRaises(FileNotFoundError):