diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 6b12ca9e58..6bc3316e27 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -1,15 +1,22 @@ # Copyright The IETF Trust 2025, All Rights Reserved import datetime +from pathlib import Path from typing import Literal, Optional +from django.db import transaction from django.urls import reverse as urlreverse from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from ietf.doc.models import DocumentAuthor, Document, RfcAuthor -from ietf.doc.utils import default_consensus +from ietf.doc.expire import move_draft_files_to_archive +from ietf.doc.models import DocumentAuthor, Document, RfcAuthor, RelatedDocument, State, \ + DocEvent +from ietf.doc.utils import default_consensus, prettify_std_name, update_action_holders +from ietf.group.models import Group +from ietf.name.models import StreamName, StdLevelName, FormalLanguageName from ietf.person.models import Person +from ietf.utils import log class PersonSerializer(serializers.ModelSerializer): @@ -196,7 +203,7 @@ class Meta: class AuthorSerializer(serializers.ModelSerializer): """Serialize an RfcAuthor record - + todo fix naming confusion with ietf.doc.serializers.RfcAuthorSerializer """ class Meta: @@ -210,3 +217,358 @@ class Meta: "affiliation", "country", ] + + +class RfcPubSerializer(serializers.ModelSerializer): + # publication-related fields + published = serializers.DateTimeField(default_timezone=datetime.timezone.utc) + draft_name = serializers.RegexField( + required=False, regex=r"^draft-[a-zA-Z0-9-]+$" + ) + draft_rev = serializers.RegexField( + required=False, regex=r"^[0-9][0-9]$" + ) + + # fields on the RFC Document that need tweaking from ModelSerializer defaults + rfc_number = serializers.IntegerField(min_value=1, required=True) + group = serializers.SlugRelatedField( + slug_field="acronym", queryset=Group.objects.all(), required=False + ) + stream = serializers.PrimaryKeyRelatedField( + queryset=StreamName.objects.filter(used=True) + ) + formal_languages = serializers.PrimaryKeyRelatedField( + many=True, + required=False, + queryset=FormalLanguageName.objects.filter(used=True), + help_text=( + "formal languages used in RFC (defaults to those from draft, send empty" + "list to override)" + ) + ) + std_level = serializers.PrimaryKeyRelatedField( + queryset=StdLevelName.objects.filter(used=True), + ) + ad = serializers.PrimaryKeyRelatedField( + queryset=Person.objects.all(), + allow_null=True, + required=False, + ) + obsoletes = serializers.SlugRelatedField( + many=True, + required=False, + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + ) + updates = serializers.SlugRelatedField( + many=True, + required=False, + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + ) + subseries = serializers.ListField( + child=serializers.RegexField( + required=False, + # pattern: no leading 0, finite length (arbitrarily set to 5 digits) + regex=r"^(bcp|std|fyi)[1-9][0-9]{0,4}$", + ) + ) + authors = AuthorSerializer(many=True) + + class Meta: + model = Document + fields = [ + "published", + "draft_name", + "draft_rev", + "rfc_number", + "title", + "authors", + "group", + "stream", + "abstract", + "pages", + "words", + "formal_languages", + "std_level", + "ad", + "note", + "obsoletes", + "updates", + "subseries", + ] + + def validate(self, data): + if "draft_name" in data or "draft_rev" in data: + if "draft_name" not in data: + raise serializers.ValidationError( + {"draft_name": "Missing draft_name"}, + code="invalid-draft-spec", + ) + if "draft_rev" not in data: + raise serializers.ValidationError( + {"draft_rev": "Missing draft_rev"}, + code="invalid-draft-spec", + ) + return data + + def create(self, validated_data): + """Publish an RFC""" + published = validated_data.pop("published") + draft_name = validated_data.pop("draft_name", None) + draft_rev = validated_data.pop("draft_rev", None) + obsoletes = validated_data.pop("obsoletes", []) + updates = validated_data.pop("updates", []) + subseries = validated_data.pop("subseries", []) + + system_person = Person.objects.get(name="(System)") + + # If specified, retrieve draft and extract RFC default values from it + if draft_name is None: + draft = None + defaults_from_draft = { + "group": Group.objects.get(acronym="none", type_id="individ"), + } + else: + # validation enforces that draft_name and draft_rev are both present + draft = Document.objects.filter( + type_id="draft", + name=draft_name, + rev=draft_rev, + ).exclude( + states__type_id="draft", + states__slug="rfc", + ).first() + if draft is None: + raise serializers.ValidationError( + { + "draft_name": "No such draft or draft already published as RFC", + "draft_rev": "No such draft or draft already published as RFC", + }, + code="invalid-draft" + ) + defaults_from_draft = { + "ad": draft.ad, + "formal_languages": draft.formal_languages.all(), + "group": draft.group, + "note": draft.note, + } + + # Transaction to clean up if something fails + with transaction.atomic(): + # create rfc, letting validated request data override draft defaults + rfc = self._create_rfc(defaults_from_draft | validated_data) + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="published_rfc", + time=published, + by=system_person, + desc="RFC published", + ) + rfc.set_state(State.objects.get(used=True, type_id="rfc", slug="published")) + + # create updates / obsoletes relations + for obsoleted_rfc_pk in obsoletes: + RelatedDocument.objects.get_or_create( + source=rfc, target=obsoleted_rfc_pk, relationship_id="obs" + ) + for updated_rfc_pk in updates: + RelatedDocument.objects.get_or_create( + source=rfc, target=updated_rfc_pk, relationship_id="updates" + ) + + # create subseries relations + for subseries_doc_name in subseries: + ss_slug = subseries_doc_name[:3] + subseries_doc, ss_doc_created = Document.objects.get_or_create( + type_id=ss_slug, name=subseries_doc_name + ) + if ss_doc_created: + subseries_doc.docevent_set.create( + type=f"{ss_slug}_doc_created", + by=system_person, + desc=f"Created {subseries_doc_name} via publication of {rfc.name}", + ) + _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create( + relationship_id="contains", target=rfc + ) + if ss_rel_created: + subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + + + # create relation with draft and update draft state + if draft is not None: + draft_changes = [] + draft_events = [] + if draft.get_state_slug() != "rfc": + draft.set_state( + State.objects.get(used=True, type="draft", slug="rfc") + ) + move_draft_files_to_archive(draft, draft.rev) + draft_changes.append(f"changed state to {draft.get_state()}") + + r, created_relateddoc = RelatedDocument.objects.get_or_create( + source=draft, target=rfc, relationship_id="became_rfc", + ) + if created_relateddoc: + change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format( + rel_name=r.relationship.name.lower(), + pretty_draft_name=prettify_std_name(draft_name), + pretty_rfc_name=prettify_std_name(rfc.name), + ) + draft_changes.append(change) + + # Always set the "draft-iesg" state. This state should be set for all drafts, so + # log a warning if it is not set. What should happen here is that ietf stream + # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index. + # Other stream documents should normally be "idexists" and be left that way. The + # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub", + # and changes any other state to "pub". If unset, it changes it to "idexists". + # This reflects historical behavior and should probably be updated, but a migration + # of existing drafts (and validation of the change) is needed before we change the + # handling. + prev_iesg_state = draft.get_state("draft-iesg") + if prev_iesg_state is None: + log.log(f'Warning while processing {rfc.name}: {draft.name} has no "draft-iesg" state') + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists") + elif prev_iesg_state.slug not in ("pub", "idexists"): + if prev_iesg_state.slug != "rfcqueue": + log.log( + 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format( + rfc.name, draft.name, prev_iesg_state.slug + ) + ) + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub") + else: + new_iesg_state = prev_iesg_state + + if new_iesg_state != prev_iesg_state: + draft.set_state(new_iesg_state) + draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}") + e = update_action_holders(draft, prev_iesg_state, new_iesg_state) + if e: + draft_events.append(e) + + # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain. + if draft.stream != rfc.stream: + log.log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format( + rfc.name, draft.name, draft.stream, rfc.stream + )) + elif draft.stream.slug in ["iab", "irtf", "ise", "editorial"]: + stream_slug = f"draft-stream-{draft.stream.slug}" + prev_state = draft.get_state(stream_slug) + if prev_state is not None and prev_state.slug != "pub": + new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub") + draft.set_state(new_state) + draft_changes.append( + f"changed {new_state.type.label} to {new_state}" + ) + e = update_action_holders(draft, prev_state, new_state) + if e: + draft_events.append(e) + if draft_changes: + draft_events.append( + DocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=system_person, + type="sync_from_rfc_editor", + desc=f"Updated while publishing {rfc.name} ({', '.join(draft_changes)})", + ) + ) + draft.save_with_history(draft_events) + + return rfc + + def _create_rfc(self, validated_data): + authors_data = validated_data.pop("authors") + formal_languages = validated_data.pop("formal_languages", []) + # todo ad field + rfc = Document.objects.create( + type_id="rfc", + name=f"rfc{validated_data['rfc_number']}", + **validated_data, + ) + rfc.formal_languages.set(formal_languages) # list of PKs is ok + for order, author_data in enumerate(authors_data): + rfc.rfcauthor_set.create( + order=order, + **author_data, + ) + return rfc + + +class RfcFileSerializer(serializers.Serializer): + # The structure of this serializer is constrained by what openapi-generator-cli's + # python generator can correctly serialize as multipart/form-data. It does not + # handle nested serializers well (or perhaps at all). ListFields with child + # ChoiceField or RegexField do not serialize correctly. DictFields don't seem + # to work. + # + # It does seem to correctly send filenames along with FileFields, even as a child + # in a ListField, so we use that to convey the file format of each item. There + # are other options we could consider (e.g., a structured CharField) but this + # works. + allowed_extensions = (".xml", ".txt", ".html", ".txt.pdf") + + rfc = serializers.SlugRelatedField( + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + help_text="RFC number to which the contents belong", + ) + contents = serializers.ListField( + child=serializers.FileField( + allow_empty_file=False, + use_url=False, + ), + help_text=( + "List of content files. Filename extensions are used to identify " + "file types, but filenames are otherwise ignored." + ), + ) + replace = serializers.BooleanField( + required=False, + default=False, + help_text=( + "Replace existing files for this RFC. Defaults to false. When false, " + "if _any_ files already exist for the specified RFC the upload will be " + "rejected regardless of which files are being uploaded. When true," + "existing files will be removed and new ones will be put in place. BE" + "VERY CAREFUL WITH THIS OPTION IN PRODUCTION." + ), + ) + + def validate_contents(self, data): + found_extensions = [] + for uploaded_file in data: + if not hasattr(uploaded_file, "name"): + raise serializers.ValidationError( + "filename not specified for uploaded file", + code="missing-filename", + ) + ext = "".join(Path(uploaded_file.name).suffixes) + if ext not in self.allowed_extensions: + raise serializers.ValidationError( + f"File uploaded with invalid extension '{ext}'", + code="invalid-filename-ext", + ) + if ext in found_extensions: + raise serializers.ValidationError( + f"More than one file uploaded with extension '{ext}'", + code="duplicate-filename-ext", + ) + return data + + +class NotificationAckSerializer(serializers.Serializer): + message = serializers.CharField(default="ack") diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index a37b37b117..9b0b871931 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -28,6 +28,8 @@ urlpatterns = [ url(r"^doc/drafts_by_names/", views_rpc.DraftsByNamesView.as_view()), url(r"^persons/search/", views_rpc.RpcPersonSearch.as_view()), + path(r"rfc/publish/", views_rpc.RfcPubNotificationView.as_view()), + path(r"rfc/publish/files/", views_rpc.RfcPubFilesView.as_view()), path(r"subject//person/", views_rpc.SubjectPersonView.as_view()), ] diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 8e183af768..7e5feed6e6 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,9 +1,14 @@ # Copyright The IETF Trust 2023-2025, All Rights Reserved +import shutil +from pathlib import Path +from tempfile import TemporaryDirectory +from django.conf import settings +from django.core.files.uploadedfile import UploadedFile from drf_spectacular.utils import OpenApiParameter -from rest_framework import serializers, viewsets, mixins +from rest_framework import mixins, parsers, serializers, viewsets, status from rest_framework.decorators import action -from rest_framework.exceptions import NotFound +from rest_framework.exceptions import NotFound, APIException from rest_framework.views import APIView from rest_framework.response import Response @@ -27,11 +32,18 @@ EmailPersonSerializer, RfcWithAuthorsSerializer, DraftWithAuthorsSerializer, + NotificationAckSerializer, RfcPubSerializer, RfcFileSerializer, ) from ietf.doc.models import Document, DocHistory, RfcAuthor from ietf.person.models import Email, Person +class Conflict(APIException): + status_code = status.HTTP_409_CONFLICT + default_detail = "Conflict." + default_code = "conflict" + + @extend_schema_view( retrieve=extend_schema( operation_id="get_person_by_id", @@ -345,3 +357,79 @@ def perform_create(self, serializer): .get("max_order") ) serializer.save(document=rfc, order=max_order + 1) + + +class RfcPubNotificationView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="notify_rfc_published", + summary="Notify datatracker of RFC publication", + request=RfcPubSerializer, + responses=NotificationAckSerializer, + ) + def post(self, request): + serializer = RfcPubSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + # Create RFC + serializer.save() + return Response(NotificationAckSerializer().data) + + +class RfcPubFilesView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + parser_classes = [parsers.MultiPartParser] + + @extend_schema( + operation_id="upload_rfc_files", + summary="Upload files for a published RFC", + request=RfcFileSerializer, + responses=NotificationAckSerializer, + ) + def post(self, request): + serializer = RfcFileSerializer( + # many=True, + data=request.data, + ) + serializer.is_valid(raise_exception=True) + rfc = serializer.validated_data["rfc"] + uploaded_files: list[UploadedFile] = serializer.validated_data["contents"] + replace = serializer.validated_data["replace"] + dest_stem = f"rfc{rfc.rfc_number}" + dest_path = Path(settings.RFC_PATH) + + # List of files that might exist for an RFC + possible_rfc_files = [ + (dest_path / dest_stem).with_suffix(ext) + for ext in serializer.allowed_extensions + ] + if not replace: + # this is the default: refuse to overwrite anything if not replacing + for possible_existing_file in possible_rfc_files: + if possible_existing_file.exists(): + raise Conflict( + "File(s) already exist for this RFC", + code="files-exist", + ) + + with TemporaryDirectory() as tempdir: + # save files with desired names in a temporary directory + files_to_move: list[Path] = [] + tmpfile_stem = Path(tempdir) / dest_stem + for upfile in uploaded_files: + uploaded_filename = Path(upfile.name) + uploaded_ext = "".join(uploaded_filename.suffixes) + dest_filename = tmpfile_stem.with_suffix(uploaded_ext) + with dest_filename.open("wb") as dest: + for chunk in upfile.chunks(): + dest.write(chunk) + files_to_move.append(dest_filename) + # copy files to final location, removing any existing ones first if the + # remove flag was set + if replace: + for possible_existing_file in possible_rfc_files: + possible_existing_file.unlink(missing_ok=True) + for ftm in files_to_move: + shutil.move(ftm, dest_path) + + return Response(NotificationAckSerializer().data)