diff --git a/ietf/api/routers.py b/ietf/api/routers.py index 745ddaa811a..cdaec153789 100644 --- a/ietf/api/routers.py +++ b/ietf/api/routers.py @@ -3,14 +3,29 @@ from django.core.exceptions import ImproperlyConfigured from rest_framework import routers -class PrefixedSimpleRouter(routers.SimpleRouter): - """SimpleRouter that adds a dot-separated prefix to its basename""" + +class PrefixedBasenameMixin: + """Mixin to add a prefix to the basename of a rest_framework BaseRouter""" def __init__(self, name_prefix="", *args, **kwargs): self.name_prefix = name_prefix if len(self.name_prefix) == 0 or self.name_prefix[-1] == ".": raise ImproperlyConfigured("Cannot use a name_prefix that is empty or ends with '.'") super().__init__(*args, **kwargs) - def get_default_basename(self, viewset): - basename = super().get_default_basename(viewset) - return f"{self.name_prefix}.{basename}" + def register(self, prefix, viewset, basename=None): + # Get the superclass "register" method from the class this is mixed-in with. + # This avoids typing issues with calling super().register() directly in a + # mixin class. + super_register = getattr(super(), "register") + if not super_register or not callable(super_register): + raise TypeError("Must mixin with superclass that has register() method") + super_register(prefix, viewset, basename=f"{self.name_prefix}.{basename}") + + +class PrefixedSimpleRouter(PrefixedBasenameMixin, routers.SimpleRouter): + """SimpleRouter that adds a dot-separated prefix to its basename""" + + +class PrefixedDefaultRouter(PrefixedBasenameMixin, routers.DefaultRouter): + """SimpleRouter that adds a dot-separated prefix to its basename""" + diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 0c6ff4c23a0..9dd78fbfe53 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -545,7 +545,14 @@ class RfcFileSerializer(serializers.Serializer): # in a ListField, so we use that to convey the file format of each item. There # are other options we could consider (e.g., a structured CharField) but this # works. - allowed_extensions = (".xml", ".txt", ".html", ".txt.pdf") + allowed_extensions = ( + ".html", + ".json", + ".notprepped.xml", + ".pdf", + ".txt", + ".xml", + ) rfc = serializers.SlugRelatedField( slug_field="rfc_number", diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 37e4416b674..032b4b9495b 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -1,37 +1,45 @@ # Copyright The IETF Trust 2025, All Rights Reserved -# -*- coding: utf-8 -*- +from io import StringIO +from pathlib import Path +from tempfile import TemporaryDirectory +from django.conf import settings +from django.core.files.base import ContentFile +from django.db.models import Max from django.test.utils import override_settings from django.urls import reverse as urlreverse -from ietf.doc.factories import IndividualDraftFactory -from ietf.doc.models import RelatedDocument -from ietf.utils.test_utils import TestCase, reload_db_objects +from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory +from ietf.doc.models import RelatedDocument, Document +from ietf.group.factories import RoleFactory, GroupFactory +from ietf.person.factories import PersonFactory +from ietf.utils.test_utils import APITestCase, reload_db_objects -class RpcApiTests(TestCase): +class RpcApiTests(APITestCase): @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) - def test_api_refs(self): + def test_draftviewset_references(self): + viewname = "ietf.api.purple_api.draft-references" + # non-existent draft - url = urlreverse("ietf.api.views_rpc.rpc_draft_refs", kwargs={"doc_id": 999999}) + bad_id = Document.objects.aggregate(unused_id=Max("id") + 100)["unused_id"] + url = urlreverse(viewname, kwargs={"doc_id": bad_id}) + # Without credentials r = self.client.get(url) - self.assertEqual(r.status_code, 200) + self.assertEqual(r.status_code, 403) + # Add credentials r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) - jsondata = r.json() - refs = jsondata["references"] - self.assertEqual(refs, []) + self.assertEqual(r.status_code, 404) # draft without any normative references draft = IndividualDraftFactory() draft = reload_db_objects(draft) - url = urlreverse( - "ietf.api.views_rpc.rpc_draft_refs", kwargs={"doc_id": draft.id} - ) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) r = self.client.get(url) - self.assertEqual(r.status_code, 200) + self.assertEqual(r.status_code, 403) r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) - jsondata = r.json() - refs = jsondata["references"] + self.assertEqual(r.status_code, 200) + refs = r.json() self.assertEqual(refs, []) # draft without any normative references but with an informative reference @@ -40,14 +48,12 @@ def test_api_refs(self): RelatedDocument.objects.create( source=draft, target=draft_foo, relationship_id="refinfo" ) - url = urlreverse( - "ietf.api.views_rpc.rpc_draft_refs", kwargs={"doc_id": draft.id} - ) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) r = self.client.get(url) - self.assertEqual(r.status_code, 200) + self.assertEqual(r.status_code, 403) r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) - jsondata = r.json() - refs = jsondata["references"] + self.assertEqual(r.status_code, 200) + refs = r.json() self.assertEqual(refs, []) # draft with a normative reference @@ -56,14 +62,238 @@ def test_api_refs(self): RelatedDocument.objects.create( source=draft, target=draft_bar, relationship_id="refnorm" ) - url = urlreverse( - "ietf.api.views_rpc.rpc_draft_refs", kwargs={"doc_id": draft.id} - ) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) r = self.client.get(url) - self.assertEqual(r.status_code, 200) + self.assertEqual(r.status_code, 403) r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) - jsondata = r.json() - refs = jsondata["references"] + self.assertEqual(r.status_code, 200) + refs = r.json() self.assertEqual(len(refs), 1) self.assertEqual(refs[0]["id"], draft_bar.id) self.assertEqual(refs[0]["name"], draft_bar.name) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + def test_notify_rfc_published(self): + url = urlreverse("ietf.api.purple_api.notify_rfc_published") + area = GroupFactory(type_id="area") + draft_ad = RoleFactory(group=area, name_id="ad").person + authors = PersonFactory.create_batch(2) + draft = WgDraftFactory(group__parent=area, authors=authors) + assert isinstance(draft, Document), "WgDraftFactory should generate a Document" + unused_rfc_number = ( + Document.objects.filter(rfc_number__isnull=False).aggregate( + unused_rfc_number=Max("rfc_number") + 1 + )["unused_rfc_number"] + or 10000 + ) + + post_data = { + "published": "2025-12-17T20:29:00Z", + "draft_name": draft.name, + "draft_rev": draft.rev, + "rfc_number": unused_rfc_number, + "title": draft.title, + "authors": [ + { + "titlepage_name": f"titlepage {author.name}", + "is_editor": False, + "person": author.pk, + "email": author.email_address(), + "affiliation": "Some Affiliation", + "country": "CA", + } + for author in authors + ], + "group": draft.group.acronym, + "stream": draft.stream_id, + "abstract": draft.abstract, + "pages": draft.pages, + "words": draft.pages * 250, + "formal_languages": [], + "std_level": "ps", + "ad": draft_ad.pk, + "note": "noted", + "obsoletes": [], + "updates": [], + "subseries": [], + } + r = self.client.post(url, data=post_data, format="json") + self.assertEqual(r.status_code, 403) + + r = self.client.post( + url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} + ) + self.assertEqual(r.status_code, 200) + rfc = Document.objects.filter(rfc_number=unused_rfc_number).first() + self.assertIsNotNone(rfc) + self.assertEqual(rfc.came_from_draft(), draft) + self.assertEqual( + rfc.docevent_set.filter( + type="published_rfc", time="2025-12-17T20:29:00Z" + ).count(), + 1, + ) + self.assertEqual(rfc.title, draft.title) + self.assertEqual(rfc.documentauthor_set.count(), 0) + self.assertEqual( + list( + rfc.rfcauthor_set.values( + "titlepage_name", + "is_editor", + "person", + "email", + "affiliation", + "country", + ) + ), + [ + { + "titlepage_name": f"titlepage {author.name}", + "is_editor": False, + "person": author.pk, + "email": author.email_address(), + "affiliation": "Some Affiliation", + "country": "CA", + } + for author in authors + ], + ) + self.assertEqual(rfc.group, draft.group) + self.assertEqual(rfc.stream, draft.stream) + self.assertEqual(rfc.abstract, draft.abstract) + self.assertEqual(rfc.pages, draft.pages) + self.assertEqual(rfc.words, draft.pages * 250) + self.assertEqual(rfc.formal_languages.count(), 0) + self.assertEqual(rfc.std_level_id, "ps") + self.assertEqual(rfc.ad, draft_ad) + self.assertEqual(rfc.note, "noted") + self.assertEqual(rfc.related_that_doc("obs"), []) + self.assertEqual(rfc.related_that_doc("updates"), []) + self.assertEqual(rfc.part_of(), []) + self.assertEqual(draft.get_state().slug, "rfc") + # todo test non-empty relationships + # todo test references (when updating that is part of the handling) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + def test_upload_rfc_files(self): + def _valid_post_data(): + """Generate a valid post data dict + + Each API call needs a fresh set of files, so don't reuse the return + value from this for multiple calls! + """ + return { + "rfc": rfc.rfc_number, + "contents": [ + ContentFile(b"This is .xml", "myfile.xml"), + ContentFile(b"This is .txt", "myfile.txt"), + ContentFile(b"This is .html", "myfile.html"), + ContentFile(b"This is .pdf", "myfile.pdf"), + ContentFile(b"This is .json", "myfile.json"), + ContentFile(b"This is .notprepped.xml", "myfile.notprepped.xml"), + ], + "replace": False, + } + + url = urlreverse("ietf.api.purple_api.upload_rfc_files") + unused_rfc_number = ( + Document.objects.filter(rfc_number__isnull=False).aggregate( + unused_rfc_number=Max("rfc_number") + 1 + )["unused_rfc_number"] + or 10000 + ) + + rfc = WgRfcFactory(rfc_number=unused_rfc_number) + assert isinstance(rfc, Document), "WgRfcFactory should generate a Document" + with TemporaryDirectory() as rfc_dir: + settings.RFC_PATH = rfc_dir # affects overridden settings + rfc_path = Path(rfc_dir) + (rfc_path / "prerelease").mkdir() + content = StringIO("XML content\n") + content.name = "myrfc.xml" + + # no api key + r = self.client.post(url, _valid_post_data(), format="multipart") + self.assertEqual(r.status_code, 403) + + # invalid RFC + r = self.client.post( + url, + _valid_post_data() | {"rfc": unused_rfc_number + 1}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + + # empty files + r = self.client.post( + url, + _valid_post_data() | { + "contents": [ + ContentFile(b"", "myfile.xml"), + ContentFile(b"", "myfile.txt"), + ContentFile(b"", "myfile.html"), + ContentFile(b"", "myfile.pdf"), + ContentFile(b"", "myfile.json"), + ContentFile(b"", "myfile.notprepped.xml"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + + # bad file type + r = self.client.post( + url, + _valid_post_data() | { + "contents": [ + ContentFile(b"Some content", "myfile.jpg"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + + # valid post + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + for suffix in [".xml", ".txt", ".html", ".pdf", ".json"]: + self.assertEqual( + (rfc_path / f"rfc{unused_rfc_number}") + .with_suffix(suffix) + .read_text(), + f"This is {suffix}", + f"{suffix} file should contain the expected content", + ) + self.assertEqual( + ( + rfc_path / "prerelease" / f"rfc{unused_rfc_number}.notprepped.xml" + ).read_text(), + "This is .notprepped.xml", + ".notprepped.xml file should contain the expected content", + ) + + # re-post with replace = False should now fail + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + + # re-post with replace = True should succeed + r = self.client.post( + url, + _valid_post_data() | {"replace": True}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) # conflict diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index 9b0b871931c..11bbc49d502 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -1,14 +1,12 @@ # Copyright The IETF Trust 2023-2025, All Rights Reserved - -from rest_framework import routers - from django.conf import settings from django.urls import include, path from ietf.api import views_rpc, views_rpc_demo +from ietf.api.routers import PrefixedDefaultRouter from ietf.utils.urls import url -router = routers.DefaultRouter(use_regex_path=False) +router = PrefixedDefaultRouter(use_regex_path=False, name_prefix="ietf.api.purple_api") router.include_format_suffixes = False router.register(r"draft", views_rpc.DraftViewSet, basename="draft") router.register(r"person", views_rpc.PersonViewSet) @@ -28,8 +26,16 @@ urlpatterns = [ url(r"^doc/drafts_by_names/", views_rpc.DraftsByNamesView.as_view()), url(r"^persons/search/", views_rpc.RpcPersonSearch.as_view()), - path(r"rfc/publish/", views_rpc.RfcPubNotificationView.as_view()), - path(r"rfc/publish/files/", views_rpc.RfcPubFilesView.as_view()), + path( + r"rfc/publish/", + views_rpc.RfcPubNotificationView.as_view(), + name="ietf.api.purple_api.notify_rfc_published", + ), + path( + r"rfc/publish/files/", + views_rpc.RfcPubFilesView.as_view(), + name="ietf.api.purple_api.upload_rfc_files", + ), path(r"subject//person/", views_rpc.SubjectPersonView.as_view()), ] diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index bec2f227680..c5ad8cbe70e 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -8,7 +8,7 @@ from drf_spectacular.utils import OpenApiParameter from rest_framework import mixins, parsers, serializers, viewsets, status from rest_framework.decorators import action -from rest_framework.exceptions import NotFound, APIException +from rest_framework.exceptions import APIException from rest_framework.views import APIView from rest_framework.response import Response @@ -34,7 +34,7 @@ NotificationAckSerializer, RfcPubSerializer, RfcFileSerializer, EditableRfcSerializer, ) -from ietf.doc.models import Document, DocHistory, RfcAuthor, EditedRfcAuthorsDocEvent +from ietf.doc.models import Document, DocHistory, RfcAuthor from ietf.doc.serializers import RfcAuthorSerializer from ietf.person.models import Email, Person @@ -367,6 +367,18 @@ class RfcPubFilesView(APIView): api_key_endpoint = "ietf.api.views_rpc" parser_classes = [parsers.MultiPartParser] + def _destination(self, filename: str | Path) -> Path: + """Destination for an uploaded RFC file + + Strips any path components in filename and returns an absolute Path. + """ + rfc_path = Path(settings.RFC_PATH) + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + return rfc_path / "prerelease" / filename.name + return rfc_path / filename.name + @extend_schema( operation_id="upload_rfc_files", summary="Upload files for a published RFC", @@ -383,11 +395,10 @@ def post(self, request): uploaded_files: list[UploadedFile] = serializer.validated_data["contents"] replace = serializer.validated_data["replace"] dest_stem = f"rfc{rfc.rfc_number}" - dest_path = Path(settings.RFC_PATH) # List of files that might exist for an RFC possible_rfc_files = [ - (dest_path / dest_stem).with_suffix(ext) + self._destination(dest_stem + ext) for ext in serializer.allowed_extensions ] if not replace: @@ -400,23 +411,25 @@ def post(self, request): ) with TemporaryDirectory() as tempdir: - # save files with desired names in a temporary directory + # Save files in a temporary directory. Use the uploaded filename + # extensions to identify files, but ignore the stems and generate our own. files_to_move: list[Path] = [] tmpfile_stem = Path(tempdir) / dest_stem for upfile in uploaded_files: - uploaded_filename = Path(upfile.name) + uploaded_filename = Path(upfile.name) # name supplied by request uploaded_ext = "".join(uploaded_filename.suffixes) - dest_filename = tmpfile_stem.with_suffix(uploaded_ext) - with dest_filename.open("wb") as dest: + tempfile_path = tmpfile_stem.with_suffix(uploaded_ext) + with tempfile_path.open("wb") as dest: for chunk in upfile.chunks(): dest.write(chunk) - files_to_move.append(dest_filename) + files_to_move.append(tempfile_path) # copy files to final location, removing any existing ones first if the # remove flag was set if replace: for possible_existing_file in possible_rfc_files: possible_existing_file.unlink(missing_ok=True) for ftm in files_to_move: - shutil.move(ftm, dest_path) + shutil.move(ftm, self._destination(ftm)) + # todo store in blob storage as well (need a bucket for RFCs) return Response(NotificationAckSerializer().data) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 2eac012c6ac..90ab877bc74 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -10,7 +10,6 @@ from rest_framework.permissions import BasePermission from rest_framework.viewsets import GenericViewSet -from drf_spectacular.utils import extend_schema from ietf.group.models import Group from ietf.name.models import StreamName, DocTypeName from ietf.utils.timezone import RPC_TZINFO diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index d9524c246b0..a946cd780b1 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -19,7 +19,6 @@ from django.conf import settings from django.contrib import messages from django.core.cache import caches -from django.db import transaction from django.db.models import OuterRef from django.forms import ValidationError from django.http import Http404 diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index 8017bb129c5..a2ab680df6d 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -27,7 +27,7 @@ from django.shortcuts import get_object_or_404 from ietf.dbtemplate.models import DBTemplate -from ietf.doc.models import DocEvent, NewRevisionDocEvent, State, Document +from ietf.doc.models import DocEvent, NewRevisionDocEvent, Document from ietf.group.models import Group, Role from ietf.person.models import Email, Person from ietf.mailtrigger.utils import gather_address_lists @@ -35,7 +35,7 @@ from ietf.meeting.utils import participants_for_meeting from ietf.utils.pipe import pipe from ietf.utils.mail import send_mail_text, send_mail, get_payload_text -from ietf.utils.log import assertion, log +from ietf.utils.log import log from ietf.person.name import unidecode_name from ietf.utils.timezone import date_today, datetime_from_date, DEADLINE_TZINFO diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py index 86c5a0c1c31..5faf83d93ff 100644 --- a/ietf/utils/test_utils.py +++ b/ietf/utils/test_utils.py @@ -38,6 +38,7 @@ import re import email import html5lib +import rest_framework.test import requests_mock import shutil import sys @@ -312,3 +313,11 @@ def tearDown(self): shutil.rmtree(dir) self.requests_mock.stop() super().tearDown() + + +class APITestCase(TestCase): + """Test case that uses rest_framework's APIClient + + This is equivalent to rest_framework.test.APITestCase, but picks up our + """ + client_class = rest_framework.test.APIClient