diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 2223f04aeb..f2e735be7a 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -5,6 +5,7 @@ from django.db import transaction from django.urls import reverse as urlreverse +from django.utils import timezone from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field from rest_framework import serializers @@ -571,6 +572,12 @@ class RfcFileSerializer(serializers.Serializer): "file types, but filenames are otherwise ignored." ), ) + mtime = serializers.DateTimeField( + required=False, + default=timezone.now, + default_timezone=datetime.UTC, + help_text="Modification timestamp to apply to uploaded files", + ) replace = serializers.BooleanField( required=False, default=False, diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 032b4b9495..a4bc456331 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -9,6 +9,7 @@ from django.test.utils import override_settings from django.urls import reverse as urlreverse +from ietf.blobdb.models import Blob from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory from ietf.doc.models import RelatedDocument, Document from ietf.group.factories import RoleFactory, GroupFactory @@ -256,6 +257,31 @@ def _valid_post_data(): ) self.assertEqual(r.status_code, 400) + # Put a file in the way. Post should fail because replace = False + file_in_the_way = (rfc_path / f"rfc{unused_rfc_number}.txt") + file_in_the_way.touch() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + file_in_the_way.unlink() + + # Put a blob in the way. Post should fail because replace = False + blob_in_the_way = Blob.objects.create( + bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b"" + ) + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + blob_in_the_way.delete() + # valid post r = self.client.post( url, @@ -264,21 +290,41 @@ def _valid_post_data(): headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 200) - for suffix in [".xml", ".txt", ".html", ".pdf", ".json"]: + for extension in ["xml", "txt", "html", "pdf", "json"]: + filename = f"rfc{unused_rfc_number}.{extension}" self.assertEqual( - (rfc_path / f"rfc{unused_rfc_number}") - .with_suffix(suffix) + (rfc_path / filename) .read_text(), - f"This is {suffix}", - f"{suffix} file should contain the expected content", + f"This is .{extension}", + f"{extension} file should contain the expected content", + ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"{extension}/{filename}" + ).content + ), + f"This is .{extension}".encode("utf-8"), + f"{extension} blob should contain the expected content", ) + # special case for notprepped + notprepped_fn = f"rfc{unused_rfc_number}.notprepped.xml" self.assertEqual( ( - rfc_path / "prerelease" / f"rfc{unused_rfc_number}.notprepped.xml" + rfc_path / "prerelease" / notprepped_fn ).read_text(), "This is .notprepped.xml", ".notprepped.xml file should contain the expected content", ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"notprepped/{notprepped_fn}" + ).content + ), + b"This is .notprepped.xml", + ".notprepped.xml blob should contain the expected content", + ) # re-post with replace = False should now fail r = self.client.post( diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index fce174ab72..542836a857 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,4 +1,5 @@ # Copyright The IETF Trust 2023-2026, All Rights Reserved +import os import shutil from pathlib import Path from tempfile import TemporaryDirectory @@ -35,6 +36,7 @@ ) from ietf.doc.models import Document, DocHistory, RfcAuthor from ietf.doc.serializers import RfcAuthorSerializer +from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage from ietf.person.models import Email, Person @@ -366,8 +368,8 @@ class RfcPubFilesView(APIView): api_key_endpoint = "ietf.api.views_rpc" parser_classes = [parsers.MultiPartParser] - def _destination(self, filename: str | Path) -> Path: - """Destination for an uploaded RFC file + def _fs_destination(self, filename: str | Path) -> Path: + """Destination for an uploaded RFC file in the filesystem Strips any path components in filename and returns an absolute Path. """ @@ -378,6 +380,23 @@ def _destination(self, filename: str | Path) -> Path: return rfc_path / "prerelease" / filename.name return rfc_path / filename.name + def _blob_destination(self, filename: str | Path) -> str: + """Destination name for an uploaded RFC file in the blob store + + Strips any path components in filename and returns an absolute Path. + """ + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + file_type = "notprepped" + elif extension[0] == ".": + file_type = extension[1:] + else: + raise serializers.ValidationError( + f"Extension does not begin with '.'!? ({filename})", + ) + return f"{file_type}/{filename.name}" + @extend_schema( operation_id="upload_rfc_files", summary="Upload files for a published RFC", @@ -394,10 +413,17 @@ def post(self, request): uploaded_files = serializer.validated_data["contents"] # list[UploadedFile] replace = serializer.validated_data["replace"] dest_stem = f"rfc{rfc.rfc_number}" + mtime = serializer.validated_data["mtime"] + mtimestamp = mtime.timestamp() + blob_kind = "rfc" # List of files that might exist for an RFC possible_rfc_files = [ - self._destination(dest_stem + ext) + self._fs_destination(dest_stem + ext) + for ext in serializer.allowed_extensions + ] + possible_rfc_blobs = [ + self._blob_destination(dest_stem + ext) for ext in serializer.allowed_extensions ] if not replace: @@ -408,6 +434,14 @@ def post(self, request): "File(s) already exist for this RFC", code="files-exist", ) + for possible_existing_blob in possible_rfc_blobs: + if exists_in_storage( + kind=blob_kind, name=possible_existing_blob + ): + raise Conflict( + "Blob(s) already exist for this RFC", + code="blobs-exist", + ) with TemporaryDirectory() as tempdir: # Save files in a temporary directory. Use the uploaded filename @@ -421,14 +455,27 @@ def post(self, request): with tempfile_path.open("wb") as dest: for chunk in upfile.chunks(): dest.write(chunk) + os.utime(tempfile_path, (mtimestamp, mtimestamp)) files_to_move.append(tempfile_path) # copy files to final location, removing any existing ones first if the # remove flag was set if replace: for possible_existing_file in possible_rfc_files: possible_existing_file.unlink(missing_ok=True) + for possible_existing_blob in possible_rfc_blobs: + remove_from_storage( + blob_kind, possible_existing_blob, warn_if_missing=False + ) for ftm in files_to_move: - shutil.move(ftm, self._destination(ftm)) - # todo store in blob storage as well (need a bucket for RFCs) + with ftm.open("rb") as f: + store_file( + kind=blob_kind, + name=self._blob_destination(ftm), + file=f, + doc_name=rfc.name, + doc_rev=rfc.rev, # expect None, but match whatever it is + mtime=mtime, + ) + shutil.move(ftm, self._fs_destination(ftm)) return Response(NotificationAckSerializer().data) diff --git a/ietf/settings.py b/ietf/settings.py index fedd313ca0..fd8d86a1ab 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -811,6 +811,7 @@ def skip_unreadable_post(record): "polls", "procmaterials", "review", + "rfc", "slides", "staging", "statchg",