From 013e720adb9edb39376f024f3a9358ebba339c4a Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 18 Nov 2025 19:04:34 -0400 Subject: [PATCH 01/21] feat: API to publish RFC (WIP) Incomplete and in need of refactoring, but publishes an RFC. --- ietf/api/serializers_rpc.py | 303 +++++++++++++++++++++++++++++++++++- ietf/api/urls_rpc.py | 1 + ietf/api/views_rpc.py | 24 +++ 3 files changed, 325 insertions(+), 3 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 6b12ca9e58..b6ead7a2de 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -7,9 +7,13 @@ from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from ietf.doc.models import DocumentAuthor, Document, RfcAuthor -from ietf.doc.utils import default_consensus +from ietf.doc.expire import move_draft_files_to_archive +from ietf.doc.models import DocumentAuthor, Document, RfcAuthor, RelatedDocument, State, \ + DocEvent +from ietf.doc.utils import default_consensus, prettify_std_name, update_action_holders +from ietf.name.models import StreamName, StdLevelName from ietf.person.models import Person +from ietf.utils import log class PersonSerializer(serializers.ModelSerializer): @@ -196,7 +200,7 @@ class Meta: class AuthorSerializer(serializers.ModelSerializer): """Serialize an RfcAuthor record - + todo fix naming confusion with ietf.doc.serializers.RfcAuthorSerializer """ class Meta: @@ -210,3 +214,296 @@ class Meta: "affiliation", "country", ] + + +class RfcPubSerializer(serializers.ModelSerializer): + # publication-related fields + published = serializers.DateTimeField(default_timezone=datetime.timezone.utc) + draft_name = serializers.RegexField( + required=False, regex=r"^draft-[a-zA-Z0-9-]+$" + ) + draft_rev = serializers.RegexField( + required=False, regex=r"^[0-9][0-9]$" + ) + + # fields on the RFC Document that need tweaking from ModelSerializer defaults + rfc_number = serializers.IntegerField(min_value=1, required=True) + stream = serializers.PrimaryKeyRelatedField( + queryset=StreamName.objects.filter(used=True) + ) + # group = serializers.SlugRelatedField( + # slug_field="acronym", + # required=False, + # queryset=Group.objects.all(), + # ) + # formal_languages = serializers.PrimaryKeyRelatedField( + # many=True, + # required=False, + # queryset=FormalLanguageName.objects.filter(used=True), + # ) + std_level = serializers.PrimaryKeyRelatedField( + queryset=StdLevelName.objects.filter(used=True), + ) + ad = serializers.PrimaryKeyRelatedField( + queryset=Person.objects.all(), + required=False, + ) + authors = AuthorSerializer(many=True) + + class Meta: + model = Document + fields = [ + "published", + "draft_name", + "draft_rev", + "rfc_number", + "title", + "authors", + "stream", + "abstract", + "pages", + "words", + # "group", + # "formal_languages", + "std_level", + "ad", + "external_url", + ] + + def validate(self, data): + if "draft_name" in data or "draft_rev" in data: + if "draft_name" not in data: + raise serializers.ValidationError( + {"draft_name": "Missing draft_name"}, + code="invalid-draft-spec", + ) + if "draft_rev" not in data: + raise serializers.ValidationError( + {"draft_rev": "Missing draft_rev"}, + code="invalid-draft-spec", + ) + return data + + def create(self, validated_data): + """Publish an RFC""" + published = validated_data.pop("published") + draft_name = validated_data.pop("draft_name", None) + draft_rev = validated_data.pop("draft_rev", None) + + # Retrieve draft + draft = None + if draft_name is not None: + # validation enforces that draft_name and draft_rev are both present + draft = Document.objects.filter( + type_id="draft", name=draft_name, rev=draft_rev, + ).first() + if draft is None: + raise serializers.ValidationError( + { + "draft_name": "No such draft", + "draft_rev": "No such draft", + }, + code="invalid-draft" + ) + # todo check that draft is in the right state + + rfc = self._create_rfc(validated_data) + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="published_rfc", + time=published, + by=Person.objects.get(name="(System)"), + desc="RFC published", + ) + rfc.set_state(State.objects.get(used=True, type_id="rfc", slug="published")) + + if draft is not None: + draft_changes = [] + draft_events = [] + if draft.get_state_slug() != "rfc": + draft.set_state( + State.objects.get(used=True, type="draft", slug="rfc") + ) + move_draft_files_to_archive(draft, draft.rev) + draft_changes.append(f"changed state to {draft.get_state()}") + + r, created_relateddoc = RelatedDocument.objects.get_or_create( + source=draft, target=rfc, relationship_id="became_rfc", + ) + if created_relateddoc: + change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format( + rel_name=r.relationship.name.lower(), + pretty_draft_name=prettify_std_name(draft_name), + pretty_rfc_name=prettify_std_name(rfc.name), + ) + draft_changes.append(change) + + # Always set the "draft-iesg" state. This state should be set for all drafts, so + # log a warning if it is not set. What should happen here is that ietf stream + # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index. + # Other stream documents should normally be "idexists" and be left that way. The + # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub", + # and changes any other state to "pub". If unset, it changes it to "idexists". + # This reflects historical behavior and should probably be updated, but a migration + # of existing drafts (and validation of the change) is needed before we change the + # handling. + prev_iesg_state = draft.get_state("draft-iesg") + if prev_iesg_state is None: + log.log(f'Warning while processing {rfc.name}: {draft.name} has no "draft-iesg" state') + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists") + elif prev_iesg_state.slug not in ("pub", "idexists"): + if prev_iesg_state.slug != "rfcqueue": + log.log( + 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format( + rfc.name, draft.name, prev_iesg_state.slug + ) + ) + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub") + else: + new_iesg_state = prev_iesg_state + + if new_iesg_state != prev_iesg_state: + draft.set_state(new_iesg_state) + draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}") + e = update_action_holders(draft, prev_iesg_state, new_iesg_state) + if e: + draft_events.append(e) + + # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain. + if draft.stream != rfc.stream: + log.log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format( + rfc.name, draft.name, draft.stream, rfc.stream + )) + elif draft.stream.slug in ["iab", "irtf", "ise"]: + stream_slug = f"draft-stream-{draft.stream.slug}" + prev_state = draft.get_state(stream_slug) + if prev_state is not None and prev_state.slug != "pub": + new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub") + draft.set_state(new_state) + draft_changes.append( + f"changed {new_state.type.label} to {new_state}" + ) + e = update_action_holders(draft, prev_state, new_state) + if e: + draft_events.append(e) + if draft_changes: + draft_events.append( + DocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=Person.objects.get(name="(System)"), + type="sync_from_rfc_editor", + desc=f"Updated while publishing {rfc.name} ({', '.join(draft_changes)})", + ) + ) + draft.save_with_history(draft_events) + + # todo set group properly + # doc.group = Group.objects.get( + # type="individ" + # ) # fallback for newly created doc + + + # todo add obsoletes / updates + # todo add subseries relationships / clean up + # todo adjust errata tags + # todo formal languages from draft + return rfc + + def _create_rfc(self, validated_data): + authors_data = validated_data.pop("authors") + formal_languages = validated_data.pop("formal_languages", []) + rfc = Document.objects.create( + type_id="rfc", + name=f"rfc{validated_data['rfc_number']}", + **validated_data, + ) + rfc.formal_languages.set(formal_languages) # list of PKs is ok + for order, author_data in enumerate(authors_data): + rfc.rfcauthor_set.create( + order=order, + **author_data, + ) + return rfc + +class RfcFileSerializer(serializers.Serializer): + filename = serializers.CharField(allow_blank=False) + content = serializers.FileField( + allow_empty_file=False, + use_url=False, + ) + + +class RfcPubNotificationSerializer(serializers.Serializer): + published = serializers.DateTimeField(default_timezone=datetime.timezone.utc) + draft_name = serializers.RegexField( + required=False, regex=r"^draft-[a-zA-Z0-9-]+$" + ) + draft_rev = serializers.RegexField( + required=False, regex=r"^[0-9][0-9]$" + ) + rfc = RfcPubSerializer() + + def validate(self, data): + if "draft_name" in data or "draft_rev" in data: + if "draft_name" not in data: + raise serializers.ValidationError( + {"draft_name": "Missing draft_name"}, + code="invalid-draft-spec", + ) + if "draft_rev" not in data: + raise serializers.ValidationError( + {"draft_rev": "Missing draft_rev"}, + code="invalid-draft-spec", + ) + return data + + def create(self, validated_data): + # Retrieve draft + draft = None + if "draft_name" in validated_data: + # serializer enforces that draft_name and draft_rev are both present + draft = Document.objects.filter( + type_id="draft", + name=validated_data["draft_name"], + rev=validated_data["draft_rev"], + ).first() + if draft is None: + raise serializers.ValidationError( + { + "draft_name": "No such draft", + "draft_rev": "No such draft", + }, + code="invalid-draft" + ) + rfc = self._create_rfc(self.validated_data.pop("rfc")) + # todo add "Created RFC" to events + # todo set doc state + # todo adjust draft state + # todo add became_rfc relationship + # todo move draft files to archive + # todo set stream state to pub + # todo set published date + # todo add obsoletes / updates + # todo add subseries relationships / clean up + # todo adjust errata tags + + def _create_rfc(self, validated_data): + authors_data = validated_data.pop("authors") + rfc = Document.objects.create( + type_id="rfc", + name=f"rfc{validated_data['rfc_number']}", + **validated_data, + ) + for order, author_data in enumerate(authors_data): + rfc.rfcauthor_set.create( + order=order, + **author_data, + ) + + + + +class NotificationAckSerializer(serializers.Serializer): + message = serializers.CharField(default="ack") diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index a37b37b117..38a5383898 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -28,6 +28,7 @@ urlpatterns = [ url(r"^doc/drafts_by_names/", views_rpc.DraftsByNamesView.as_view()), url(r"^persons/search/", views_rpc.RpcPersonSearch.as_view()), + path(r"rfc/publish/", views_rpc.RfcPubNotificationView.as_view()), path(r"subject//person/", views_rpc.SubjectPersonView.as_view()), ] diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 8e183af768..8a48ab990d 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -27,6 +27,7 @@ EmailPersonSerializer, RfcWithAuthorsSerializer, DraftWithAuthorsSerializer, + NotificationAckSerializer, RfcPubSerializer, ) from ietf.doc.models import Document, DocHistory, RfcAuthor from ietf.person.models import Email, Person @@ -345,3 +346,26 @@ def perform_create(self, serializer): .get("max_order") ) serializer.save(document=rfc, order=max_order + 1) + + +class RfcPubNotificationView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="notify_rfc_published", + summary="Notify datatracker of RFC publication", + request=RfcPubSerializer, + responses=NotificationAckSerializer, + ) + def post(self, request): + print(request.POST) # todo remove debug + serializer = RfcPubSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + # Create RFC + serializer.save() + + print(">>> Notified of RFC publication!!") + from pprint import pp + pp(serializer.validated_data) + return Response(NotificationAckSerializer().data) From b572f7df0045d17c937afa4fe583c9ead36aa413 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Nov 2025 09:20:50 -0400 Subject: [PATCH 02/21] feat: group / formal_languages from draft --- ietf/api/serializers_rpc.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index b6ead7a2de..acf503f19d 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -307,7 +307,12 @@ def create(self, validated_data): ) # todo check that draft is in the right state - rfc = self._create_rfc(validated_data) + rfc = self._create_rfc( + validated_data | { + "group": draft.group if draft else "none", + "formal_languages": draft.formal_languages.all() if draft else [], + } + ) DocEvent.objects.create( doc=rfc, rev=rfc.rev, @@ -399,16 +404,8 @@ def create(self, validated_data): ) draft.save_with_history(draft_events) - # todo set group properly - # doc.group = Group.objects.get( - # type="individ" - # ) # fallback for newly created doc - - # todo add obsoletes / updates # todo add subseries relationships / clean up - # todo adjust errata tags - # todo formal languages from draft return rfc def _create_rfc(self, validated_data): From 1c15b8f2ce3293d043738a56a14163ee4826aa6f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Nov 2025 09:26:11 -0400 Subject: [PATCH 03/21] feat: allow optional formal_languages via API Could do the same with group, but not clear it would ever be used. --- ietf/api/serializers_rpc.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index acf503f19d..260458a448 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -11,7 +11,7 @@ from ietf.doc.models import DocumentAuthor, Document, RfcAuthor, RelatedDocument, State, \ DocEvent from ietf.doc.utils import default_consensus, prettify_std_name, update_action_holders -from ietf.name.models import StreamName, StdLevelName +from ietf.name.models import StreamName, StdLevelName, FormalLanguageName from ietf.person.models import Person from ietf.utils import log @@ -231,16 +231,15 @@ class RfcPubSerializer(serializers.ModelSerializer): stream = serializers.PrimaryKeyRelatedField( queryset=StreamName.objects.filter(used=True) ) - # group = serializers.SlugRelatedField( - # slug_field="acronym", - # required=False, - # queryset=Group.objects.all(), - # ) - # formal_languages = serializers.PrimaryKeyRelatedField( - # many=True, - # required=False, - # queryset=FormalLanguageName.objects.filter(used=True), - # ) + formal_languages = serializers.PrimaryKeyRelatedField( + many=True, + required=False, + queryset=FormalLanguageName.objects.filter(used=True), + help_text=( + "formal languages used in RFC (defaults to those from draft, send empty" + "list to override)" + ) + ) std_level = serializers.PrimaryKeyRelatedField( queryset=StdLevelName.objects.filter(used=True), ) @@ -263,8 +262,7 @@ class Meta: "abstract", "pages", "words", - # "group", - # "formal_languages", + "formal_languages", "std_level", "ad", "external_url", @@ -308,10 +306,10 @@ def create(self, validated_data): # todo check that draft is in the right state rfc = self._create_rfc( - validated_data | { + { "group": draft.group if draft else "none", "formal_languages": draft.formal_languages.all() if draft else [], - } + } | validated_data ) DocEvent.objects.create( doc=rfc, From 4cff06ccf6b054bfc232936e16e3809b3f31be5a Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Nov 2025 14:58:43 -0400 Subject: [PATCH 04/21] feat: fill in overrides/updates --- ietf/api/serializers_rpc.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 260458a448..bc308a10f0 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -247,6 +247,16 @@ class RfcPubSerializer(serializers.ModelSerializer): queryset=Person.objects.all(), required=False, ) + obsoletes = serializers.PrimaryKeyRelatedField( + many=True, + required=False, + queryset=Document.objects.filter(type_id="rfc"), + ) + updates = serializers.PrimaryKeyRelatedField( + many=True, + required=False, + queryset=Document.objects.filter(type_id="rfc"), + ) authors = AuthorSerializer(many=True) class Meta: @@ -266,6 +276,8 @@ class Meta: "std_level", "ad", "external_url", + "obsoletes", + "updates", ] def validate(self, data): @@ -287,7 +299,9 @@ def create(self, validated_data): published = validated_data.pop("published") draft_name = validated_data.pop("draft_name", None) draft_rev = validated_data.pop("draft_rev", None) - + obsoletes = validated_data.pop("obsoletes", []) + updates = validated_data.pop("updates", []) + # Retrieve draft draft = None if draft_name is not None: @@ -321,6 +335,17 @@ def create(self, validated_data): ) rfc.set_state(State.objects.get(used=True, type_id="rfc", slug="published")) + # create updates / obsoletes relations + for obsoleted_rfc_pk in obsoletes: + RelatedDocument.objects.create( + source=rfc, target=obsoleted_rfc_pk, relationship_id="obs" + ) + for updated_rfc_pk in updates: + RelatedDocument.objects.create( + source=rfc, target=updated_rfc_pk, relationship_id="updates" + ) + + # create relation with draft and update draft state if draft is not None: draft_changes = [] draft_events = [] @@ -402,7 +427,6 @@ def create(self, validated_data): ) draft.save_with_history(draft_events) - # todo add obsoletes / updates # todo add subseries relationships / clean up return rfc From 543dd991aa2604bb25659b9d08a6dbc203e1e296 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Nov 2025 15:49:06 -0400 Subject: [PATCH 05/21] feat: subseries membership --- ietf/api/serializers_rpc.py | 41 +++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index bc308a10f0..97bc3f5d21 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -257,6 +257,13 @@ class RfcPubSerializer(serializers.ModelSerializer): required=False, queryset=Document.objects.filter(type_id="rfc"), ) + subseries = serializers.ListField( + child=serializers.RegexField( + required=False, + # pattern: no leading 0, finite length (arbitrarily set to 5 digits) + regex=r"^(bcp|std|fyi)[1-9][0-9]{0,4}$", + ) + ) authors = AuthorSerializer(many=True) class Meta: @@ -278,6 +285,7 @@ class Meta: "external_url", "obsoletes", "updates", + "subseries", ] def validate(self, data): @@ -301,6 +309,7 @@ def create(self, validated_data): draft_rev = validated_data.pop("draft_rev", None) obsoletes = validated_data.pop("obsoletes", []) updates = validated_data.pop("updates", []) + subseries = validated_data.pop("subseries", []) # Retrieve draft draft = None @@ -319,6 +328,7 @@ def create(self, validated_data): ) # todo check that draft is in the right state + system_person = Person.objects.get(name="(System)") rfc = self._create_rfc( { "group": draft.group if draft else "none", @@ -330,7 +340,7 @@ def create(self, validated_data): rev=rfc.rev, type="published_rfc", time=published, - by=Person.objects.get(name="(System)"), + by=system_person, desc="RFC published", ) rfc.set_state(State.objects.get(used=True, type_id="rfc", slug="published")) @@ -345,6 +355,33 @@ def create(self, validated_data): source=rfc, target=updated_rfc_pk, relationship_id="updates" ) + # create subseries relations + for subseries_doc_name in subseries: + ss_slug = subseries_doc_name[:3] + subseries_doc, created = Document.objects.get_or_create( + type_id=ss_slug, name=subseries_doc_name + ) + if created: + subseries_doc.docevent_set.create( + type=f"{ss_slug}_doc_created", + by=system_person, + desc=f"Created {subseries_doc_name} via publication of {rfc.name}", + ) + subseries_doc.relateddocument_set.create( + relationship_id="contains", target=rfc + ) + subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + + # create relation with draft and update draft state if draft is not None: draft_changes = [] @@ -420,7 +457,7 @@ def create(self, validated_data): DocEvent.objects.create( doc=draft, rev=draft.rev, - by=Person.objects.get(name="(System)"), + by=system_person, type="sync_from_rfc_editor", desc=f"Updated while publishing {rfc.name} ({', '.join(draft_changes)})", ) From c91bb9b60f13955a595dfbc6bde860c0f21bd024 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Nov 2025 16:22:35 -0400 Subject: [PATCH 06/21] fix: tolerate race to create related docs --- ietf/api/serializers_rpc.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 97bc3f5d21..7b83d55293 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -347,39 +347,40 @@ def create(self, validated_data): # create updates / obsoletes relations for obsoleted_rfc_pk in obsoletes: - RelatedDocument.objects.create( + RelatedDocument.objects.get_or_create( source=rfc, target=obsoleted_rfc_pk, relationship_id="obs" ) for updated_rfc_pk in updates: - RelatedDocument.objects.create( + RelatedDocument.objects.get_or_create( source=rfc, target=updated_rfc_pk, relationship_id="updates" ) # create subseries relations for subseries_doc_name in subseries: ss_slug = subseries_doc_name[:3] - subseries_doc, created = Document.objects.get_or_create( + subseries_doc, ss_doc_created = Document.objects.get_or_create( type_id=ss_slug, name=subseries_doc_name ) - if created: + if ss_doc_created: subseries_doc.docevent_set.create( type=f"{ss_slug}_doc_created", by=system_person, desc=f"Created {subseries_doc_name} via publication of {rfc.name}", ) - subseries_doc.relateddocument_set.create( + _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create( relationship_id="contains", target=rfc ) - subseries_doc.docevent_set.create( - type="sync_from_rfc_editor", - by=system_person, - desc=f"Added {rfc.name} to {subseries_doc.name}", - ) - rfc.docevent_set.create( - type="sync_from_rfc_editor", - by=system_person, - desc=f"Added {rfc.name} to {subseries_doc.name}", - ) + if ss_rel_created: + subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) # create relation with draft and update draft state From 626e0b2a2855150845f5fe030d26bdde329df95d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Nov 2025 16:28:06 -0400 Subject: [PATCH 07/21] fix: wrap pub in a transaction --- ietf/api/serializers_rpc.py | 249 ++++++++++++++++++------------------ 1 file changed, 126 insertions(+), 123 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 7b83d55293..50b2e7bd72 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -2,6 +2,7 @@ import datetime from typing import Literal, Optional +from django.db import transaction from django.urls import reverse as urlreverse from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field @@ -328,142 +329,144 @@ def create(self, validated_data): ) # todo check that draft is in the right state - system_person = Person.objects.get(name="(System)") - rfc = self._create_rfc( - { - "group": draft.group if draft else "none", - "formal_languages": draft.formal_languages.all() if draft else [], - } | validated_data - ) - DocEvent.objects.create( - doc=rfc, - rev=rfc.rev, - type="published_rfc", - time=published, - by=system_person, - desc="RFC published", - ) - rfc.set_state(State.objects.get(used=True, type_id="rfc", slug="published")) - - # create updates / obsoletes relations - for obsoleted_rfc_pk in obsoletes: - RelatedDocument.objects.get_or_create( - source=rfc, target=obsoleted_rfc_pk, relationship_id="obs" + # Transaction to clean up if something fails + with transaction.atomic(): + system_person = Person.objects.get(name="(System)") + rfc = self._create_rfc( + { + "group": draft.group if draft else "none", + "formal_languages": draft.formal_languages.all() if draft else [], + } | validated_data ) - for updated_rfc_pk in updates: - RelatedDocument.objects.get_or_create( - source=rfc, target=updated_rfc_pk, relationship_id="updates" + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="published_rfc", + time=published, + by=system_person, + desc="RFC published", ) + rfc.set_state(State.objects.get(used=True, type_id="rfc", slug="published")) - # create subseries relations - for subseries_doc_name in subseries: - ss_slug = subseries_doc_name[:3] - subseries_doc, ss_doc_created = Document.objects.get_or_create( - type_id=ss_slug, name=subseries_doc_name - ) - if ss_doc_created: - subseries_doc.docevent_set.create( - type=f"{ss_slug}_doc_created", - by=system_person, - desc=f"Created {subseries_doc_name} via publication of {rfc.name}", + # create updates / obsoletes relations + for obsoleted_rfc_pk in obsoletes: + RelatedDocument.objects.get_or_create( + source=rfc, target=obsoleted_rfc_pk, relationship_id="obs" ) - _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create( - relationship_id="contains", target=rfc - ) - if ss_rel_created: - subseries_doc.docevent_set.create( - type="sync_from_rfc_editor", - by=system_person, - desc=f"Added {rfc.name} to {subseries_doc.name}", - ) - rfc.docevent_set.create( - type="sync_from_rfc_editor", - by=system_person, - desc=f"Added {rfc.name} to {subseries_doc.name}", + for updated_rfc_pk in updates: + RelatedDocument.objects.get_or_create( + source=rfc, target=updated_rfc_pk, relationship_id="updates" ) - - - # create relation with draft and update draft state - if draft is not None: - draft_changes = [] - draft_events = [] - if draft.get_state_slug() != "rfc": - draft.set_state( - State.objects.get(used=True, type="draft", slug="rfc") + + # create subseries relations + for subseries_doc_name in subseries: + ss_slug = subseries_doc_name[:3] + subseries_doc, ss_doc_created = Document.objects.get_or_create( + type_id=ss_slug, name=subseries_doc_name ) - move_draft_files_to_archive(draft, draft.rev) - draft_changes.append(f"changed state to {draft.get_state()}") - - r, created_relateddoc = RelatedDocument.objects.get_or_create( - source=draft, target=rfc, relationship_id="became_rfc", - ) - if created_relateddoc: - change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format( - rel_name=r.relationship.name.lower(), - pretty_draft_name=prettify_std_name(draft_name), - pretty_rfc_name=prettify_std_name(rfc.name), + if ss_doc_created: + subseries_doc.docevent_set.create( + type=f"{ss_slug}_doc_created", + by=system_person, + desc=f"Created {subseries_doc_name} via publication of {rfc.name}", + ) + _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create( + relationship_id="contains", target=rfc ) - draft_changes.append(change) - - # Always set the "draft-iesg" state. This state should be set for all drafts, so - # log a warning if it is not set. What should happen here is that ietf stream - # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index. - # Other stream documents should normally be "idexists" and be left that way. The - # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub", - # and changes any other state to "pub". If unset, it changes it to "idexists". - # This reflects historical behavior and should probably be updated, but a migration - # of existing drafts (and validation of the change) is needed before we change the - # handling. - prev_iesg_state = draft.get_state("draft-iesg") - if prev_iesg_state is None: - log.log(f'Warning while processing {rfc.name}: {draft.name} has no "draft-iesg" state') - new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists") - elif prev_iesg_state.slug not in ("pub", "idexists"): - if prev_iesg_state.slug != "rfcqueue": - log.log( - 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format( - rfc.name, draft.name, prev_iesg_state.slug - ) + if ss_rel_created: + subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", ) - new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub") - else: - new_iesg_state = prev_iesg_state + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + - if new_iesg_state != prev_iesg_state: - draft.set_state(new_iesg_state) - draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}") - e = update_action_holders(draft, prev_iesg_state, new_iesg_state) - if e: - draft_events.append(e) - - # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain. - if draft.stream != rfc.stream: - log.log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format( - rfc.name, draft.name, draft.stream, rfc.stream - )) - elif draft.stream.slug in ["iab", "irtf", "ise"]: - stream_slug = f"draft-stream-{draft.stream.slug}" - prev_state = draft.get_state(stream_slug) - if prev_state is not None and prev_state.slug != "pub": - new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub") - draft.set_state(new_state) - draft_changes.append( - f"changed {new_state.type.label} to {new_state}" + # create relation with draft and update draft state + if draft is not None: + draft_changes = [] + draft_events = [] + if draft.get_state_slug() != "rfc": + draft.set_state( + State.objects.get(used=True, type="draft", slug="rfc") ) - e = update_action_holders(draft, prev_state, new_state) + move_draft_files_to_archive(draft, draft.rev) + draft_changes.append(f"changed state to {draft.get_state()}") + + r, created_relateddoc = RelatedDocument.objects.get_or_create( + source=draft, target=rfc, relationship_id="became_rfc", + ) + if created_relateddoc: + change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format( + rel_name=r.relationship.name.lower(), + pretty_draft_name=prettify_std_name(draft_name), + pretty_rfc_name=prettify_std_name(rfc.name), + ) + draft_changes.append(change) + + # Always set the "draft-iesg" state. This state should be set for all drafts, so + # log a warning if it is not set. What should happen here is that ietf stream + # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index. + # Other stream documents should normally be "idexists" and be left that way. The + # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub", + # and changes any other state to "pub". If unset, it changes it to "idexists". + # This reflects historical behavior and should probably be updated, but a migration + # of existing drafts (and validation of the change) is needed before we change the + # handling. + prev_iesg_state = draft.get_state("draft-iesg") + if prev_iesg_state is None: + log.log(f'Warning while processing {rfc.name}: {draft.name} has no "draft-iesg" state') + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists") + elif prev_iesg_state.slug not in ("pub", "idexists"): + if prev_iesg_state.slug != "rfcqueue": + log.log( + 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format( + rfc.name, draft.name, prev_iesg_state.slug + ) + ) + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub") + else: + new_iesg_state = prev_iesg_state + + if new_iesg_state != prev_iesg_state: + draft.set_state(new_iesg_state) + draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}") + e = update_action_holders(draft, prev_iesg_state, new_iesg_state) if e: draft_events.append(e) - if draft_changes: - draft_events.append( - DocEvent.objects.create( - doc=draft, - rev=draft.rev, - by=system_person, - type="sync_from_rfc_editor", - desc=f"Updated while publishing {rfc.name} ({', '.join(draft_changes)})", + + # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain. + if draft.stream != rfc.stream: + log.log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format( + rfc.name, draft.name, draft.stream, rfc.stream + )) + elif draft.stream.slug in ["iab", "irtf", "ise"]: + stream_slug = f"draft-stream-{draft.stream.slug}" + prev_state = draft.get_state(stream_slug) + if prev_state is not None and prev_state.slug != "pub": + new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub") + draft.set_state(new_state) + draft_changes.append( + f"changed {new_state.type.label} to {new_state}" + ) + e = update_action_holders(draft, prev_state, new_state) + if e: + draft_events.append(e) + if draft_changes: + draft_events.append( + DocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=system_person, + type="sync_from_rfc_editor", + desc=f"Updated while publishing {rfc.name} ({', '.join(draft_changes)})", + ) ) - ) - draft.save_with_history(draft_events) + draft.save_with_history(draft_events) # todo add subseries relationships / clean up return rfc From d773cef7a143455ec1f5ef33db6c0f3c6ae0768c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Nov 2025 16:36:55 -0400 Subject: [PATCH 08/21] feat: prevent re-publishing draft as RFC --- ietf/api/serializers_rpc.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 50b2e7bd72..546ca520c8 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -317,17 +317,21 @@ def create(self, validated_data): if draft_name is not None: # validation enforces that draft_name and draft_rev are both present draft = Document.objects.filter( - type_id="draft", name=draft_name, rev=draft_rev, + type_id="draft", + name=draft_name, + rev=draft_rev, + ).exclude( + states__type_id="draft", + states__slug="rfc", ).first() if draft is None: raise serializers.ValidationError( { - "draft_name": "No such draft", - "draft_rev": "No such draft", + "draft_name": "No such draft or draft already published as RFC", + "draft_rev": "No such draft or draft already published as RFC", }, code="invalid-draft" ) - # todo check that draft is in the right state # Transaction to clean up if something fails with transaction.atomic(): From 6cdb1bddb0dfae6cd6ca9b0211fac6c6ca2382b4 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Nov 2025 16:37:35 -0400 Subject: [PATCH 09/21] chore: remove stale code --- ietf/api/serializers_rpc.py | 71 ------------------------------------- 1 file changed, 71 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 546ca520c8..03e6d8d2ca 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -472,7 +472,6 @@ def create(self, validated_data): ) draft.save_with_history(draft_events) - # todo add subseries relationships / clean up return rfc def _create_rfc(self, validated_data): @@ -499,75 +498,5 @@ class RfcFileSerializer(serializers.Serializer): ) -class RfcPubNotificationSerializer(serializers.Serializer): - published = serializers.DateTimeField(default_timezone=datetime.timezone.utc) - draft_name = serializers.RegexField( - required=False, regex=r"^draft-[a-zA-Z0-9-]+$" - ) - draft_rev = serializers.RegexField( - required=False, regex=r"^[0-9][0-9]$" - ) - rfc = RfcPubSerializer() - - def validate(self, data): - if "draft_name" in data or "draft_rev" in data: - if "draft_name" not in data: - raise serializers.ValidationError( - {"draft_name": "Missing draft_name"}, - code="invalid-draft-spec", - ) - if "draft_rev" not in data: - raise serializers.ValidationError( - {"draft_rev": "Missing draft_rev"}, - code="invalid-draft-spec", - ) - return data - - def create(self, validated_data): - # Retrieve draft - draft = None - if "draft_name" in validated_data: - # serializer enforces that draft_name and draft_rev are both present - draft = Document.objects.filter( - type_id="draft", - name=validated_data["draft_name"], - rev=validated_data["draft_rev"], - ).first() - if draft is None: - raise serializers.ValidationError( - { - "draft_name": "No such draft", - "draft_rev": "No such draft", - }, - code="invalid-draft" - ) - rfc = self._create_rfc(self.validated_data.pop("rfc")) - # todo add "Created RFC" to events - # todo set doc state - # todo adjust draft state - # todo add became_rfc relationship - # todo move draft files to archive - # todo set stream state to pub - # todo set published date - # todo add obsoletes / updates - # todo add subseries relationships / clean up - # todo adjust errata tags - - def _create_rfc(self, validated_data): - authors_data = validated_data.pop("authors") - rfc = Document.objects.create( - type_id="rfc", - name=f"rfc{validated_data['rfc_number']}", - **validated_data, - ) - for order, author_data in enumerate(authors_data): - rfc.rfcauthor_set.create( - order=order, - **author_data, - ) - - - - class NotificationAckSerializer(serializers.Serializer): message = serializers.CharField(default="ack") From a02d2c1126d83961fac13bd8eb71a4394652de9b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 19 Nov 2025 16:54:56 -0400 Subject: [PATCH 10/21] chore: remove debug --- ietf/api/views_rpc.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 8a48ab990d..871f6b03c4 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -358,14 +358,8 @@ class RfcPubNotificationView(APIView): responses=NotificationAckSerializer, ) def post(self, request): - print(request.POST) # todo remove debug serializer = RfcPubSerializer(data=request.data) serializer.is_valid(raise_exception=True) - # Create RFC serializer.save() - - print(">>> Notified of RFC publication!!") - from pprint import pp - pp(serializer.validated_data) return Response(NotificationAckSerializer().data) From bddcfcf8f2abbd1d9554c19ba18028493e19a5a0 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 20 Nov 2025 12:25:30 -0400 Subject: [PATCH 11/21] feat: RFC file upload API (WIP) Checkpointing progress before going further. --- ietf/api/serializers_rpc.py | 35 +++++++++++++++++++++++++++++++---- ietf/api/urls_rpc.py | 1 + ietf/api/views_rpc.py | 28 ++++++++++++++++++++++++++-- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 03e6d8d2ca..762c48282a 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -7,6 +7,7 @@ from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field from rest_framework import serializers +from rest_framework.validators import UniqueValidator from ietf.doc.expire import move_draft_files_to_archive from ietf.doc.models import DocumentAuthor, Document, RfcAuthor, RelatedDocument, State, \ @@ -490,12 +491,38 @@ def _create_rfc(self, validated_data): ) return rfc + class RfcFileSerializer(serializers.Serializer): - filename = serializers.CharField(allow_blank=False) - content = serializers.FileField( - allow_empty_file=False, - use_url=False, + # The structure of this serializer is constrained by what openapi-generator-cli's + # python generator can correctly serialize as multipart/form-data. It does not + # handle nested serializers well (or perhaps at all). ListFields with child + # ChoiceField or RegexField do not serialize correctly. DictFields don't seem + # to work. + # + # It does seem to correctly send filenames along with FileFields, even as a child + # in a ListField, so we use that to convey the file format of each item. There + # are other options we could consider (e.g., a structured CharField) but this + # works. + contents = serializers.ListField( + child=serializers.FileField( + allow_empty_file=False, + use_url=False, + ), + help_text=( + "List of content files. Filename extensions are used to identify " + "file types, but filenames are otherwise ignored." + ), ) + + def validate(self, data): + if len(data["filetypes"]) != len(data["contents"]): + raise serializers.ValidationError( + { + "contents": "Number of contents does not match number of filetypes" + }, + code="contents-count-mismatch", + ) + return data class NotificationAckSerializer(serializers.Serializer): diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index 38a5383898..9b0b871931 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -29,6 +29,7 @@ url(r"^doc/drafts_by_names/", views_rpc.DraftsByNamesView.as_view()), url(r"^persons/search/", views_rpc.RpcPersonSearch.as_view()), path(r"rfc/publish/", views_rpc.RfcPubNotificationView.as_view()), + path(r"rfc/publish/files/", views_rpc.RfcPubFilesView.as_view()), path(r"subject//person/", views_rpc.SubjectPersonView.as_view()), ] diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 871f6b03c4..ba06a8ce3c 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2023-2025, All Rights Reserved from drf_spectacular.utils import OpenApiParameter -from rest_framework import serializers, viewsets, mixins +from rest_framework import mixins, parsers, serializers, viewsets from rest_framework.decorators import action from rest_framework.exceptions import NotFound from rest_framework.views import APIView @@ -27,7 +27,7 @@ EmailPersonSerializer, RfcWithAuthorsSerializer, DraftWithAuthorsSerializer, - NotificationAckSerializer, RfcPubSerializer, + NotificationAckSerializer, RfcPubSerializer, RfcFileSerializer, ) from ietf.doc.models import Document, DocHistory, RfcAuthor from ietf.person.models import Email, Person @@ -363,3 +363,27 @@ def post(self, request): # Create RFC serializer.save() return Response(NotificationAckSerializer().data) + + +class RfcPubFilesView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + parser_classes = [parsers.MultiPartParser] + + @extend_schema( + operation_id="upload_rfc_files", + summary="Upload files for a published RFC", + request=RfcFileSerializer, + responses=NotificationAckSerializer, + ) + def post(self, request): + print(request.POST) # todo remove debug + serializer = RfcFileSerializer( + # many=True, + data=request.data, + ) + serializer.is_valid(raise_exception=True) + + print(">>> Got some files") + from pprint import pp + pp(serializer.validated_data) + return Response(NotificationAckSerializer().data) From 75ad7f2986e34388b7767d5a7c04762165a497e2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 20 Nov 2025 13:04:34 -0400 Subject: [PATCH 12/21] feat: specify RFC, validate file exts --- ietf/api/serializers_rpc.py | 38 +++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 762c48282a..3fc2eb4e74 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -1,5 +1,6 @@ # Copyright The IETF Trust 2025, All Rights Reserved import datetime +from pathlib import Path from typing import Literal, Optional from django.db import transaction @@ -503,6 +504,13 @@ class RfcFileSerializer(serializers.Serializer): # in a ListField, so we use that to convey the file format of each item. There # are other options we could consider (e.g., a structured CharField) but this # works. + allowed_extensions = (".xml", ".txt", ".html", ".txt.pdf") + + rfc = serializers.SlugRelatedField( + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + help_text="RFC number to which the contents belong", + ) contents = serializers.ListField( child=serializers.FileField( allow_empty_file=False, @@ -513,16 +521,26 @@ class RfcFileSerializer(serializers.Serializer): "file types, but filenames are otherwise ignored." ), ) - - def validate(self, data): - if len(data["filetypes"]) != len(data["contents"]): - raise serializers.ValidationError( - { - "contents": "Number of contents does not match number of filetypes" - }, - code="contents-count-mismatch", - ) - return data + + def validate_contents(self, data): + found_extensions = [] + for uploaded_file in data: + if not hasattr(uploaded_file, "name"): + raise serializers.ValidationError( + "filename not specified for uploaded file", + code="missing-filename", + ) + ext = "".join(Path(uploaded_file.name).suffixes) + if ext not in self.allowed_extensions: + raise serializers.ValidationError( + f"File uploaded with invalid extension '{ext}'", + code="invalid-filename-ext", + ) + if ext in found_extensions: + raise serializers.ValidationError( + f"More than one file uploaded with extension '{ext}'", + code="duplicate-filename-ext", + ) class NotificationAckSerializer(serializers.Serializer): From 952109df1f7cdc1152fef0ac1e5fc3c19ad95ff0 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 20 Nov 2025 13:51:16 -0400 Subject: [PATCH 13/21] feat: move uploaded files into place --- ietf/api/serializers_rpc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 3fc2eb4e74..e677a4106f 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -541,6 +541,7 @@ def validate_contents(self, data): f"More than one file uploaded with extension '{ext}'", code="duplicate-filename-ext", ) + return data class NotificationAckSerializer(serializers.Serializer): From 86423d65603c1e14583f4d6484d0f49d4bfc8750 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 20 Nov 2025 14:19:54 -0400 Subject: [PATCH 14/21] feat: add replace option --- ietf/api/serializers_rpc.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index e677a4106f..ebde06a17e 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -521,6 +521,17 @@ class RfcFileSerializer(serializers.Serializer): "file types, but filenames are otherwise ignored." ), ) + replace = serializers.BooleanField( + required=False, + default=False, + help_text=( + "Replace existing files for this RFC. Defaults to false. When false, " + "if _any_ files already exist for the specified RFC the upload will be " + "rejected regardless of which files are being uploaded. When true," + "existing files will be removed and new ones will be put in place. BE" + "VERY CAREFUL WITH THIS OPTION IN PRODUCTION." + ), + ) def validate_contents(self, data): found_extensions = [] From 82d822b5b6480a80c37aa697cb4da2dc14eac70f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 20 Nov 2025 15:28:29 -0400 Subject: [PATCH 15/21] fix: add rest of replace option --- ietf/api/serializers_rpc.py | 1 - ietf/api/views_rpc.py | 58 +++++++++++++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index ebde06a17e..06b3568d70 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -8,7 +8,6 @@ from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from rest_framework.validators import UniqueValidator from ietf.doc.expire import move_draft_files_to_archive from ietf.doc.models import DocumentAuthor, Document, RfcAuthor, RelatedDocument, State, \ diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index ba06a8ce3c..7e5feed6e6 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,9 +1,14 @@ # Copyright The IETF Trust 2023-2025, All Rights Reserved +import shutil +from pathlib import Path +from tempfile import TemporaryDirectory +from django.conf import settings +from django.core.files.uploadedfile import UploadedFile from drf_spectacular.utils import OpenApiParameter -from rest_framework import mixins, parsers, serializers, viewsets +from rest_framework import mixins, parsers, serializers, viewsets, status from rest_framework.decorators import action -from rest_framework.exceptions import NotFound +from rest_framework.exceptions import NotFound, APIException from rest_framework.views import APIView from rest_framework.response import Response @@ -33,6 +38,12 @@ from ietf.person.models import Email, Person +class Conflict(APIException): + status_code = status.HTTP_409_CONFLICT + default_detail = "Conflict." + default_code = "conflict" + + @extend_schema_view( retrieve=extend_schema( operation_id="get_person_by_id", @@ -376,14 +387,49 @@ class RfcPubFilesView(APIView): responses=NotificationAckSerializer, ) def post(self, request): - print(request.POST) # todo remove debug serializer = RfcFileSerializer( # many=True, data=request.data, ) serializer.is_valid(raise_exception=True) + rfc = serializer.validated_data["rfc"] + uploaded_files: list[UploadedFile] = serializer.validated_data["contents"] + replace = serializer.validated_data["replace"] + dest_stem = f"rfc{rfc.rfc_number}" + dest_path = Path(settings.RFC_PATH) + + # List of files that might exist for an RFC + possible_rfc_files = [ + (dest_path / dest_stem).with_suffix(ext) + for ext in serializer.allowed_extensions + ] + if not replace: + # this is the default: refuse to overwrite anything if not replacing + for possible_existing_file in possible_rfc_files: + if possible_existing_file.exists(): + raise Conflict( + "File(s) already exist for this RFC", + code="files-exist", + ) + + with TemporaryDirectory() as tempdir: + # save files with desired names in a temporary directory + files_to_move: list[Path] = [] + tmpfile_stem = Path(tempdir) / dest_stem + for upfile in uploaded_files: + uploaded_filename = Path(upfile.name) + uploaded_ext = "".join(uploaded_filename.suffixes) + dest_filename = tmpfile_stem.with_suffix(uploaded_ext) + with dest_filename.open("wb") as dest: + for chunk in upfile.chunks(): + dest.write(chunk) + files_to_move.append(dest_filename) + # copy files to final location, removing any existing ones first if the + # remove flag was set + if replace: + for possible_existing_file in possible_rfc_files: + possible_existing_file.unlink(missing_ok=True) + for ftm in files_to_move: + shutil.move(ftm, dest_path) - print(">>> Got some files") - from pprint import pp - pp(serializer.validated_data) return Response(NotificationAckSerializer().data) From eb18fb969ed81da852c924fa83b27cbf64356d65 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 20 Nov 2025 17:56:15 -0400 Subject: [PATCH 16/21] feat: handle ad/group more consistently --- ietf/api/serializers_rpc.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 06b3568d70..043cb88d32 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -13,6 +13,7 @@ from ietf.doc.models import DocumentAuthor, Document, RfcAuthor, RelatedDocument, State, \ DocEvent from ietf.doc.utils import default_consensus, prettify_std_name, update_action_holders +from ietf.group.models import Group from ietf.name.models import StreamName, StdLevelName, FormalLanguageName from ietf.person.models import Person from ietf.utils import log @@ -230,6 +231,9 @@ class RfcPubSerializer(serializers.ModelSerializer): # fields on the RFC Document that need tweaking from ModelSerializer defaults rfc_number = serializers.IntegerField(min_value=1, required=True) + group = serializers.SlugRelatedField( + slug_field="acronym", queryset=Group.objects.all(), required=False + ) stream = serializers.PrimaryKeyRelatedField( queryset=StreamName.objects.filter(used=True) ) @@ -247,6 +251,7 @@ class RfcPubSerializer(serializers.ModelSerializer): ) ad = serializers.PrimaryKeyRelatedField( queryset=Person.objects.all(), + allow_null=True, required=False, ) obsoletes = serializers.PrimaryKeyRelatedField( @@ -277,6 +282,7 @@ class Meta: "rfc_number", "title", "authors", + "group", "stream", "abstract", "pages", @@ -339,8 +345,12 @@ def create(self, validated_data): system_person = Person.objects.get(name="(System)") rfc = self._create_rfc( { - "group": draft.group if draft else "none", + "ad": draft.ad if draft else None, "formal_languages": draft.formal_languages.all() if draft else [], + "group": ( + draft.group if draft else Group.objects.get(acronym="none") + ), + "shepherd": draft.shepherd if draft else None, } | validated_data ) DocEvent.objects.create( @@ -478,6 +488,7 @@ def create(self, validated_data): def _create_rfc(self, validated_data): authors_data = validated_data.pop("authors") formal_languages = validated_data.pop("formal_languages", []) + # todo ad field rfc = Document.objects.create( type_id="rfc", name=f"rfc{validated_data['rfc_number']}", From d5235eab6ae359cf2a9b1080019c8923a250e535 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 20 Nov 2025 18:18:51 -0400 Subject: [PATCH 17/21] chore: remove inadvertent change --- ietf/api/serializers_rpc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 043cb88d32..0397fcce4a 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -350,7 +350,6 @@ def create(self, validated_data): "group": ( draft.group if draft else Group.objects.get(acronym="none") ), - "shepherd": draft.shepherd if draft else None, } | validated_data ) DocEvent.objects.create( From b7fe5f8e24732bc29f51b81f9c4075da49db37d3 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 21 Nov 2025 11:18:09 -0400 Subject: [PATCH 18/21] chore: drop external_url, get note from draft --- ietf/api/serializers_rpc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 0397fcce4a..064a94b7f1 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -290,7 +290,7 @@ class Meta: "formal_languages", "std_level", "ad", - "external_url", + "note", "obsoletes", "updates", "subseries", @@ -350,6 +350,7 @@ def create(self, validated_data): "group": ( draft.group if draft else Group.objects.get(acronym="none") ), + "note": draft.note if draft else "", } | validated_data ) DocEvent.objects.create( From c01fdbc4163c4e070fe48f1580447952f5a5f3b2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 21 Nov 2025 11:25:13 -0400 Subject: [PATCH 19/21] refactor: clarify default value logic --- ietf/api/serializers_rpc.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 064a94b7f1..1a128a3603 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -319,9 +319,15 @@ def create(self, validated_data): updates = validated_data.pop("updates", []) subseries = validated_data.pop("subseries", []) - # Retrieve draft - draft = None - if draft_name is not None: + system_person = Person.objects.get(name="(System)") + + # If specified, retrieve draft and extract RFC default values from it + if draft_name is None: + draft = None + defaults_from_draft = { + "group": Group.objects.get(acronym="none", type_id="individ"), + } + else: # validation enforces that draft_name and draft_rev are both present draft = Document.objects.filter( type_id="draft", @@ -339,20 +345,17 @@ def create(self, validated_data): }, code="invalid-draft" ) + defaults_from_draft = { + "ad": draft.ad, + "formal_languages": draft.formal_languages.all(), + "group": draft.group, + "note": draft.note, + } # Transaction to clean up if something fails with transaction.atomic(): - system_person = Person.objects.get(name="(System)") - rfc = self._create_rfc( - { - "ad": draft.ad if draft else None, - "formal_languages": draft.formal_languages.all() if draft else [], - "group": ( - draft.group if draft else Group.objects.get(acronym="none") - ), - "note": draft.note if draft else "", - } | validated_data - ) + # create rfc, letting validated request data override draft defaults + rfc = self._create_rfc(defaults_from_draft | validated_data) DocEvent.objects.create( doc=rfc, rev=rfc.rev, From 950480d06cf9cb87cac2a20e64a1272bff81047f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 24 Nov 2025 14:12:25 -0400 Subject: [PATCH 20/21] refactor: ID obsoletes/updates by number --- ietf/api/serializers_rpc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 1a128a3603..650d881754 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -254,14 +254,16 @@ class RfcPubSerializer(serializers.ModelSerializer): allow_null=True, required=False, ) - obsoletes = serializers.PrimaryKeyRelatedField( + obsoletes = serializers.SlugRelatedField( many=True, required=False, + slug_field="rfc_number", queryset=Document.objects.filter(type_id="rfc"), ) - updates = serializers.PrimaryKeyRelatedField( + updates = serializers.SlugRelatedField( many=True, required=False, + slug_field="rfc_number", queryset=Document.objects.filter(type_id="rfc"), ) subseries = serializers.ListField( From 42767a47e330d5c6239d306cbe331ce73d4eccf6 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 24 Nov 2025 14:48:04 -0400 Subject: [PATCH 21/21] fix: handle draft-stream-editorial --- ietf/api/serializers_rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 650d881754..6bc3316e27 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -464,7 +464,7 @@ def create(self, validated_data): log.log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format( rfc.name, draft.name, draft.stream, rfc.stream )) - elif draft.stream.slug in ["iab", "irtf", "ise"]: + elif draft.stream.slug in ["iab", "irtf", "ise", "editorial"]: stream_slug = f"draft-stream-{draft.stream.slug}" prev_state = draft.get_state(stream_slug) if prev_state is not None and prev_state.slug != "pub":