From 5b2a64b9652d9e270f87f46b6492b8dabe0bfca0 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 13 Jan 2026 13:34:09 -0400 Subject: [PATCH 1/6] feat: set mtime for RFC pub files --- ietf/api/serializers_rpc.py | 7 +++++++ ietf/api/views_rpc.py | 3 +++ 2 files changed, 10 insertions(+) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 2223f04aeb..f2e735be7a 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -5,6 +5,7 @@ from django.db import transaction from django.urls import reverse as urlreverse +from django.utils import timezone from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field from rest_framework import serializers @@ -571,6 +572,12 @@ class RfcFileSerializer(serializers.Serializer): "file types, but filenames are otherwise ignored." ), ) + mtime = serializers.DateTimeField( + required=False, + default=timezone.now, + default_timezone=datetime.UTC, + help_text="Modification timestamp to apply to uploaded files", + ) replace = serializers.BooleanField( required=False, default=False, diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index fce174ab72..3e902b235c 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,4 +1,5 @@ # Copyright The IETF Trust 2023-2026, All Rights Reserved +import os import shutil from pathlib import Path from tempfile import TemporaryDirectory @@ -394,6 +395,7 @@ def post(self, request): uploaded_files = serializer.validated_data["contents"] # list[UploadedFile] replace = serializer.validated_data["replace"] dest_stem = f"rfc{rfc.rfc_number}" + mtime = serializer.validated_data["mtime"].timestamp() # List of files that might exist for an RFC possible_rfc_files = [ @@ -421,6 +423,7 @@ def post(self, request): with tempfile_path.open("wb") as dest: for chunk in upfile.chunks(): dest.write(chunk) + os.utime(tempfile_path, (mtime, mtime)) files_to_move.append(tempfile_path) # copy files to final location, removing any existing ones first if the # remove flag was set From ed9ca3452c1898ce381cc5fcb8c07a15674ed775 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 14 Jan 2026 11:14:39 -0400 Subject: [PATCH 2/6] chore: add rfc storage --- ietf/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ietf/settings.py b/ietf/settings.py index fedd313ca0..fd8d86a1ab 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -811,6 +811,7 @@ def skip_unreadable_post(record): "polls", "procmaterials", "review", + "rfc", "slides", "staging", "statchg", From 952b8d5185c0a8cebbcf5b194cbf116e0dc47265 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 14 Jan 2026 12:43:43 -0400 Subject: [PATCH 3/6] refactor: destination helper is fs-specific --- ietf/api/views_rpc.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 3e902b235c..8a00ff00d6 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -36,6 +36,7 @@ ) from ietf.doc.models import Document, DocHistory, RfcAuthor from ietf.doc.serializers import RfcAuthorSerializer +from ietf.doc.storage_utils import remove_from_storage from ietf.person.models import Email, Person @@ -367,8 +368,8 @@ class RfcPubFilesView(APIView): api_key_endpoint = "ietf.api.views_rpc" parser_classes = [parsers.MultiPartParser] - def _destination(self, filename: str | Path) -> Path: - """Destination for an uploaded RFC file + def _fs_destination(self, filename: str | Path) -> Path: + """Destination for an uploaded RFC file in the filesystem Strips any path components in filename and returns an absolute Path. """ @@ -399,7 +400,7 @@ def post(self, request): # List of files that might exist for an RFC possible_rfc_files = [ - self._destination(dest_stem + ext) + self._fs_destination(dest_stem + ext) for ext in serializer.allowed_extensions ] if not replace: @@ -431,7 +432,7 @@ def post(self, request): for possible_existing_file in possible_rfc_files: possible_existing_file.unlink(missing_ok=True) for ftm in files_to_move: - shutil.move(ftm, self._destination(ftm)) + shutil.move(ftm, self._fs_destination(ftm)) # todo store in blob storage as well (need a bucket for RFCs) return Response(NotificationAckSerializer().data) From 557255aa3c0f67b567ce7a5c4361b0e3e6cdbea4 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 14 Jan 2026 13:20:10 -0400 Subject: [PATCH 4/6] feat: RFC files->blobstore in publish API --- ietf/api/views_rpc.py | 50 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 8a00ff00d6..ca8e4abcff 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -36,7 +36,7 @@ ) from ietf.doc.models import Document, DocHistory, RfcAuthor from ietf.doc.serializers import RfcAuthorSerializer -from ietf.doc.storage_utils import remove_from_storage +from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage from ietf.person.models import Email, Person @@ -380,6 +380,23 @@ def _fs_destination(self, filename: str | Path) -> Path: return rfc_path / "prerelease" / filename.name return rfc_path / filename.name + def _blob_destination(self, filename: str | Path) -> str: + """Destination name for an uploaded RFC file in the blob store + + Strips any path components in filename and returns an absolute Path. + """ + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + file_type = "notprepped" + elif extension[0] == ".": + file_type = extension[1:] + else: + raise serializers.ValidationError( + f"Extension does not begin with '.'!? ({filename})", + ) + return f"{file_type}/{filename.name}" + @extend_schema( operation_id="upload_rfc_files", summary="Upload files for a published RFC", @@ -396,13 +413,19 @@ def post(self, request): uploaded_files = serializer.validated_data["contents"] # list[UploadedFile] replace = serializer.validated_data["replace"] dest_stem = f"rfc{rfc.rfc_number}" - mtime = serializer.validated_data["mtime"].timestamp() + mtime = serializer.validated_data["mtime"] + mtimestamp = mtime.timestamp() + blob_kind = "rfc" # List of files that might exist for an RFC possible_rfc_files = [ self._fs_destination(dest_stem + ext) for ext in serializer.allowed_extensions ] + possible_rfc_blobs = [ + self._blob_destination(dest_stem + ext) + for ext in serializer.allowed_extensions + ] if not replace: # this is the default: refuse to overwrite anything if not replacing for possible_existing_file in possible_rfc_files: @@ -411,6 +434,14 @@ def post(self, request): "File(s) already exist for this RFC", code="files-exist", ) + for possible_existing_blob in possible_rfc_blobs: + if exists_in_storage( + kind=blob_kind, name=possible_existing_blob + ): + raise Conflict( + "Blob(s) already exist for this RFC", + code="blobs-exist", + ) with TemporaryDirectory() as tempdir: # Save files in a temporary directory. Use the uploaded filename @@ -424,14 +455,27 @@ def post(self, request): with tempfile_path.open("wb") as dest: for chunk in upfile.chunks(): dest.write(chunk) - os.utime(tempfile_path, (mtime, mtime)) + os.utime(tempfile_path, (mtimestamp, mtimestamp)) files_to_move.append(tempfile_path) # copy files to final location, removing any existing ones first if the # remove flag was set if replace: for possible_existing_file in possible_rfc_files: possible_existing_file.unlink(missing_ok=True) + for possible_existing_blob in possible_rfc_blobs: + remove_from_storage( + blob_kind, possible_existing_blob, warn_if_missing=False + ) for ftm in files_to_move: + with ftm.open("rb") as f: + store_file( + kind=blob_kind, + name=self._blob_destination(ftm), + file=f, + doc_name=rfc.name, + doc_rev=rfc.rev, # expect None, but match whatever it is + mtime=mtime, + ) shutil.move(ftm, self._fs_destination(ftm)) # todo store in blob storage as well (need a bucket for RFCs) From fccbe7015478ac2f08186c4c543ca873df247af6 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 14 Jan 2026 18:29:25 -0400 Subject: [PATCH 5/6] test: test blob writing --- ietf/api/tests_views_rpc.py | 58 +++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 032b4b9495..a4bc456331 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -9,6 +9,7 @@ from django.test.utils import override_settings from django.urls import reverse as urlreverse +from ietf.blobdb.models import Blob from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory from ietf.doc.models import RelatedDocument, Document from ietf.group.factories import RoleFactory, GroupFactory @@ -256,6 +257,31 @@ def _valid_post_data(): ) self.assertEqual(r.status_code, 400) + # Put a file in the way. Post should fail because replace = False + file_in_the_way = (rfc_path / f"rfc{unused_rfc_number}.txt") + file_in_the_way.touch() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + file_in_the_way.unlink() + + # Put a blob in the way. Post should fail because replace = False + blob_in_the_way = Blob.objects.create( + bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b"" + ) + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + blob_in_the_way.delete() + # valid post r = self.client.post( url, @@ -264,21 +290,41 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 200) - for suffix in [".xml", ".txt", ".html", ".pdf", ".json"]: + for extension in ["xml", "txt", "html", "pdf", "json"]: + filename = f"rfc{unused_rfc_number}.{extension}" self.assertEqual( - (rfc_path / f"rfc{unused_rfc_number}") - .with_suffix(suffix) + (rfc_path / filename) .read_text(), - f"This is {suffix}", - f"{suffix} file should contain the expected content", + f"This is .{extension}", + f"{extension} file should contain the expected content", + ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"{extension}/{filename}" + ).content + ), + f"This is .{extension}".encode("utf-8"), + f"{extension} blob should contain the expected content", ) + # special case for notprepped + notprepped_fn = f"rfc{unused_rfc_number}.notprepped.xml" self.assertEqual( ( - rfc_path / "prerelease" / f"rfc{unused_rfc_number}.notprepped.xml" + rfc_path / "prerelease" / notprepped_fn ).read_text(), "This is .notprepped.xml", ".notprepped.xml file should contain the expected content", ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"notprepped/{notprepped_fn}" + ).content + ), + b"This is .notprepped.xml", + ".notprepped.xml blob should contain the expected content", + ) # re-post with replace = False should now fail r = self.client.post( From d04b1cdc68c0305c3b9cefef8ad65623df67d0b5 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 14 Jan 2026 19:02:14 -0400 Subject: [PATCH 6/6] chore: remove completed todo comment --- ietf/api/views_rpc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index ca8e4abcff..542836a857 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -477,6 +477,5 @@ def post(self, request): mtime=mtime, ) shutil.move(ftm, self._fs_destination(ftm)) - # todo store in blob storage as well (need a bucket for RFCs) return Response(NotificationAckSerializer().data)