feat: store blobs+set mtime in RFC publish API

ietf/api/serializers_rpc.py

@@ -5,6 +5,7 @@
 
 from django.db import transaction
 from django.urls import reverse as urlreverse
+from django.utils import timezone
 from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import extend_schema_field
 from rest_framework import serializers
@@ -571,6 +572,12 @@ class RfcFileSerializer(serializers.Serializer):
             "file types, but filenames are otherwise ignored."
         ),
     )
+    mtime = serializers.DateTimeField(
+        required=False,
+        default=timezone.now,
+        default_timezone=datetime.UTC,
+        help_text="Modification timestamp to apply to uploaded files",
+    )
     replace = serializers.BooleanField(
         required=False,
         default=False,

ietf/api/tests_views_rpc.py

@@ -9,6 +9,7 @@
 from django.test.utils import override_settings
 from django.urls import reverse as urlreverse
 
+from ietf.blobdb.models import Blob
 from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory
 from ietf.doc.models import RelatedDocument, Document
 from ietf.group.factories import RoleFactory, GroupFactory
@@ -256,6 +257,31 @@ def _valid_post_data():
             )
             self.assertEqual(r.status_code, 400)
 
+            # Put a file in the way. Post should fail because replace = False
+            file_in_the_way = (rfc_path / f"rfc{unused_rfc_number}.txt")
+            file_in_the_way.touch()
+            r = self.client.post(
+                url,
+                _valid_post_data(),
+                format="multipart",
+                headers={"X-Api-Key": "valid-token"},
+            )
+            self.assertEqual(r.status_code, 409)  # conflict
+            file_in_the_way.unlink()
+
+            # Put a blob in the way. Post should fail because replace = False
+            blob_in_the_way = Blob.objects.create(
+                bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b""
+            )
+            r = self.client.post(
+                url,
+                _valid_post_data(),
+                format="multipart",
+                headers={"X-Api-Key": "valid-token"},
+            )
+            self.assertEqual(r.status_code, 409)  # conflict
+            blob_in_the_way.delete()
+
             # valid post
             r = self.client.post(
                 url,
@@ -264,21 +290,41 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 200)
-            for suffix in [".xml", ".txt", ".html", ".pdf", ".json"]:
+            for extension in ["xml", "txt", "html", "pdf", "json"]:
+                filename = f"rfc{unused_rfc_number}.{extension}"
                 self.assertEqual(
-                    (rfc_path / f"rfc{unused_rfc_number}")
-                    .with_suffix(suffix)
+                    (rfc_path / filename)
                     .read_text(),
-                    f"This is {suffix}",
-                    f"{suffix} file should contain the expected content",
+                    f"This is .{extension}",
+                    f"{extension} file should contain the expected content",
+                )
+                self.assertEqual(
+                    bytes(
+                        Blob.objects.get(
+                            bucket="rfc", name=f"{extension}/{filename}"
+                        ).content
+                    ),
+                    f"This is .{extension}".encode("utf-8"),
+                    f"{extension} blob should contain the expected content",
                 )
+            # special case for notprepped
+            notprepped_fn = f"rfc{unused_rfc_number}.notprepped.xml"
             self.assertEqual(
                 (
-                    rfc_path / "prerelease" / f"rfc{unused_rfc_number}.notprepped.xml"
+                    rfc_path / "prerelease" / notprepped_fn
                 ).read_text(),
                 "This is .notprepped.xml",
                 ".notprepped.xml file should contain the expected content",
             )
+            self.assertEqual(
+                bytes(
+                    Blob.objects.get(
+                        bucket="rfc", name=f"notprepped/{notprepped_fn}"
+                    ).content
+                ),
+                b"This is .notprepped.xml",
+                ".notprepped.xml blob should contain the expected content",
+            )
 
             # re-post with replace = False should now fail
             r = self.client.post(

ietf/api/views_rpc.py

@@ -1,4 +1,5 @@
 # Copyright The IETF Trust 2023-2026, All Rights Reserved
+import os
 import shutil
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -35,6 +36,7 @@
 )
 from ietf.doc.models import Document, DocHistory, RfcAuthor
 from ietf.doc.serializers import RfcAuthorSerializer
+from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage
 from ietf.person.models import Email, Person
 
 
@@ -366,8 +368,8 @@ class RfcPubFilesView(APIView):
     api_key_endpoint = "ietf.api.views_rpc"
     parser_classes = [parsers.MultiPartParser]
 
-    def _destination(self, filename: str | Path) -> Path:
-        """Destination for an uploaded RFC file
+    def _fs_destination(self, filename: str | Path) -> Path:
+        """Destination for an uploaded RFC file in the filesystem
 
         Strips any path components in filename and returns an absolute Path.
         """
@@ -378,6 +380,23 @@ def _destination(self, filename: str | Path) -> Path:
             return rfc_path / "prerelease" / filename.name
         return rfc_path / filename.name
 
+    def _blob_destination(self, filename: str | Path) -> str:
+        """Destination name for an uploaded RFC file in the blob store
+
+        Strips any path components in filename and returns an absolute Path.
+        """
+        filename = Path(filename)  # could potentially have directory components
+        extension = "".join(filename.suffixes)
+        if extension == ".notprepped.xml":
+            file_type = "notprepped"
+        elif extension[0] == ".":
+            file_type = extension[1:]
+        else:
+            raise serializers.ValidationError(
+                f"Extension does not begin with '.'!? ({filename})",
+            )
+        return f"{file_type}/{filename.name}"
+
     @extend_schema(
         operation_id="upload_rfc_files",
         summary="Upload files for a published RFC",
@@ -394,10 +413,17 @@ def post(self, request):
         uploaded_files = serializer.validated_data["contents"]  # list[UploadedFile]
         replace = serializer.validated_data["replace"]
         dest_stem = f"rfc{rfc.rfc_number}"
+        mtime = serializer.validated_data["mtime"]
+        mtimestamp = mtime.timestamp()
+        blob_kind = "rfc"
 
         # List of files that might exist for an RFC
         possible_rfc_files = [
-            self._destination(dest_stem + ext)
+            self._fs_destination(dest_stem + ext)
+            for ext in serializer.allowed_extensions
+        ]
+        possible_rfc_blobs = [
+            self._blob_destination(dest_stem + ext)
             for ext in serializer.allowed_extensions
         ]
         if not replace:
@@ -408,6 +434,14 @@ def post(self, request):
                         "File(s) already exist for this RFC",
                         code="files-exist",
                     )
+            for possible_existing_blob in possible_rfc_blobs:
+                if exists_in_storage(
+                    kind=blob_kind, name=possible_existing_blob
+                ):
+                    raise Conflict(
+                        "Blob(s) already exist for this RFC",
+                        code="blobs-exist",
+                    )
 
         with TemporaryDirectory() as tempdir:
             # Save files in a temporary directory. Use the uploaded filename
@@ -421,14 +455,27 @@ def post(self, request):
                 with tempfile_path.open("wb") as dest:
                     for chunk in upfile.chunks():
                         dest.write(chunk)
+                os.utime(tempfile_path, (mtimestamp, mtimestamp))
                 files_to_move.append(tempfile_path)
             # copy files to final location, removing any existing ones first if the
             # remove flag was set
             if replace:
                 for possible_existing_file in possible_rfc_files:
                     possible_existing_file.unlink(missing_ok=True)
+                for possible_existing_blob in possible_rfc_blobs:
+                    remove_from_storage(
+                        blob_kind, possible_existing_blob, warn_if_missing=False
+                    )
             for ftm in files_to_move:
-                shutil.move(ftm, self._destination(ftm))
-                # todo store in blob storage as well (need a bucket for RFCs)
+                with ftm.open("rb") as f:
+                    store_file(
+                        kind=blob_kind,
+                        name=self._blob_destination(ftm),
+                        file=f,
+                        doc_name=rfc.name,
+                        doc_rev=rfc.rev,  # expect None, but match whatever it is
+                        mtime=mtime,
+                    )
+                shutil.move(ftm, self._fs_destination(ftm))
 
         return Response(NotificationAckSerializer().data)

ietf/settings.py

@@ -811,6 +811,7 @@ def skip_unreadable_post(record):
     "polls",
     "procmaterials",
     "review",
+    "rfc",
     "slides",
     "staging",
     "statchg",