chore: merge main into release

Author: jennifer-richards

.gitignore

@@ -1,6 +1,7 @@
 .DS_store
 datatracker.sublime-project
 datatracker.sublime-workspace
+/.claude
 /.coverage
 /.factoryboy_random_state
 /.mypy_cache

dev/build/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20260408T1908
+FROM ghcr.io/ietf-tools/datatracker-app-base:20260410T1557
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive

dev/build/TARGET_BASE

@@ -1 +1 @@
-20260408T1908
+20260410T1557

ietf/api/tests_views_rpc.py

@@ -1,4 +1,5 @@
 # Copyright The IETF Trust 2025, All Rights Reserved
+import datetime
 from io import StringIO
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -10,12 +11,15 @@
 from django.test.utils import override_settings
 from django.urls import reverse as urlreverse
 import mock
+from django.utils import timezone
 
 from ietf.blobdb.models import Blob
 from ietf.doc.factories import IndividualDraftFactory, RfcFactory, WgDraftFactory, WgRfcFactory
 from ietf.doc.models import RelatedDocument, Document
 from ietf.group.factories import RoleFactory, GroupFactory
 from ietf.person.factories import PersonFactory
+from ietf.sync.rfcindex import rfcindex_is_dirty
+from ietf.utils.models import DirtyBits
 from ietf.utils.test_utils import APITestCase, reload_db_objects
 
 
@@ -408,16 +412,21 @@ def _valid_post_data():
             )
 
     @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
-    @mock.patch("ietf.api.views_rpc.create_rfc_index_task")
-    def test_refresh_rfc_index(self, mock_task):
+    def test_refresh_rfc_index(self):
+        DirtyBits.objects.create(
+            slug=DirtyBits.Slugs.RFCINDEX,
+            dirty_time=timezone.now() - datetime.timedelta(days=1),
+            processed_time=timezone.now() - datetime.timedelta(hours=12),
+        )
+        self.assertFalse(rfcindex_is_dirty())
         url = urlreverse("ietf.api.purple_api.refresh_rfc_index")
         response = self.client.get(url)
         self.assertEqual(response.status_code, 403)
         response = self.client.get(url, headers={"X-Api-Key": "invalid-token"})
         self.assertEqual(response.status_code, 403)
         response = self.client.get(url, headers={"X-Api-Key": "valid-token"})
         self.assertEqual(response.status_code, 405)
-        self.assertFalse(mock_task.delay.called)
+        self.assertFalse(rfcindex_is_dirty())
         response = self.client.post(url, headers={"X-Api-Key": "valid-token"})
         self.assertEqual(response.status_code, 202)
-        self.assertTrue(mock_task.delay.called)
+        self.assertTrue(rfcindex_is_dirty())

ietf/api/views_rpc.py

@@ -32,7 +32,9 @@
     EmailPersonSerializer,
     RfcWithAuthorsSerializer,
     DraftWithAuthorsSerializer,
-    NotificationAckSerializer, RfcPubSerializer, RfcFileSerializer,
+    NotificationAckSerializer,
+    RfcPubSerializer,
+    RfcFileSerializer,
     EditableRfcSerializer,
 )
 from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent
@@ -45,7 +47,7 @@
     update_rfc_searchindex_task,
 )
 from ietf.person.models import Email, Person
-from ietf.sync.tasks import create_rfc_index_task
+from ietf.sync.rfcindex import mark_rfcindex_as_dirty
 
 
 class Conflict(APIException):
@@ -344,9 +346,10 @@ def post(self, request):
 
 class RfcAuthorViewSet(viewsets.ReadOnlyModelViewSet):
     """ViewSet for RfcAuthor model
-    
+
     Router needs to provide rfc_number as a kwarg
     """
+
     api_key_endpoint = "ietf.api.views_rpc"
 
     queryset = RfcAuthor.objects.all()
@@ -407,7 +410,7 @@ class RfcPubFilesView(APIView):
 
     def _fs_destination(self, filename: str | Path) -> Path:
         """Destination for an uploaded RFC file in the filesystem
-        
+
         Strips any path components in filename and returns an absolute Path.
         """
         rfc_path = Path(settings.RFC_PATH)
@@ -419,7 +422,7 @@ def _fs_destination(self, filename: str | Path) -> Path:
 
     def _blob_destination(self, filename: str | Path) -> str:
         """Destination name for an uploaded RFC file in the blob store
-        
+
         Strips any path components in filename and returns an absolute Path.
         """
         filename = Path(filename)  # could potentially have directory components
@@ -472,9 +475,7 @@ def post(self, request):
                         code="files-exist",
                     )
             for possible_existing_blob in possible_rfc_blobs:
-                if exists_in_storage(
-                    kind=blob_kind, name=possible_existing_blob
-                ):
+                if exists_in_storage(kind=blob_kind, name=possible_existing_blob):
                     raise Conflict(
                         "Blob(s) already exist for this RFC",
                         code="blobs-exist",
@@ -523,7 +524,9 @@ def post(self, request):
 
         # Trigger red precomputer
         needs_updating = [rfc.rfc_number]
-        for rel in rfc.relateddocument_set.filter(relationship_id__in=["obs","updates"]):
+        for rel in rfc.relateddocument_set.filter(
+            relationship_id__in=["obs", "updates"]
+        ):
             needs_updating.append(rel.target.rfc_number)
         trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating))
         # Trigger search index update
@@ -540,10 +543,10 @@ class RfcIndexView(APIView):
     @extend_schema(
         operation_id="refresh_rfc_index",
         summary="Refresh rfc-index files",
-        description="Requests creation of rfc-index.xml and rfc-index.txt files",
+        description="Requests creation of various index files.",
         responses={202: None},
         request=None,
     )
     def post(self, request):
-        create_rfc_index_task.delay()
+        mark_rfcindex_as_dirty()
         return Response(status=202)

ietf/doc/feeds.py

@@ -5,6 +5,7 @@
 import datetime
 import unicodedata
 
+from django.conf import settings
 from django.contrib.syndication.views import Feed, FeedDoesNotExist
 from django.utils.feedgenerator import Atom1Feed, Rss201rev2Feed
 from django.urls import reverse as urlreverse
@@ -223,7 +224,7 @@ def item_extra_kwargs(self, item):
         extra.update({"dcterms_accessRights": "gratis"})
         extra.update({"dcterms_format": "text/html"})
         media_contents = []
-        if item.rfc_number < 8650:
+        if item.rfc_number < settings.FIRST_V3_RFC:
             if item.rfc_number not in [8, 9, 51, 418, 500, 530, 589]:
                 for fmt, media_type in [("txt", "text/plain"), ("html", "text/html")]:
                     media_contents.append(

ietf/doc/storage_utils.py

@@ -178,8 +178,7 @@ def retrieve_bytes(kind: str, name: str) -> bytes:
                     content = f.read()
         except Exception as err:
             log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}")
-            if settings.SERVER_MODE == "development":
-                raise
+            raise
     return content
 
 
@@ -192,6 +191,5 @@ def retrieve_str(kind: str, name: str) -> str:
             content = content_bytes.decode("utf-8")
         except Exception as err:
             log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}")
-            if settings.SERVER_MODE == "development":
-                raise
+            raise
     return content

ietf/doc/tests_notprepped.py

@@ -0,0 +1,122 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+from django.conf import settings
+from django.utils import timezone
+from django.urls import reverse as urlreverse
+
+from pyquery import PyQuery
+
+from ietf.doc.factories import WgRfcFactory
+from ietf.doc.models import StoredObject
+from ietf.doc.storage_utils import store_bytes
+from ietf.utils.test_utils import TestCase
+
+
+class NotpreppedRfcXmlTests(TestCase):
+    def test_editor_source_button_visibility(self):
+        pre_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC - 1)
+        first_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC)
+        post_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC + 1)
+
+        for rfc, expect_button in [(pre_v3, False), (first_v3, True), (post_v3, True)]:
+            r = self.client.get(
+                urlreverse(
+                    "ietf.doc.views_doc.document_main", kwargs=dict(name=rfc.name)
+                )
+            )
+            self.assertEqual(r.status_code, 200)
+            buttons = PyQuery(r.content)('a.btn:contains("Get editor source")')
+            if expect_button:
+                self.assertEqual(len(buttons), 1, msg=f"rfc_number={rfc.rfc_number}")
+                expected_href = urlreverse(
+                    "ietf.doc.views_doc.rfcxml_notprepped_wrapper",
+                    kwargs=dict(number=rfc.rfc_number),
+                )
+                self.assertEqual(
+                    buttons.attr("href"),
+                    expected_href,
+                    msg=f"rfc_number={rfc.rfc_number}",
+                )
+            else:
+                self.assertEqual(len(buttons), 0, msg=f"rfc_number={rfc.rfc_number}")
+
+    def test_rfcxml_notprepped(self):
+        number = settings.FIRST_V3_RFC
+        stored_name = f"notprepped/rfc{number}.notprepped.xml"
+        url = f"/doc/rfc{number}/notprepped/"
+
+        # 404 for pre-v3 RFC numbers (no document needed)
+        r = self.client.get(f"/doc/rfc{number - 1}/notprepped/")
+        self.assertEqual(r.status_code, 404)
+
+        # 404 when no RFC document exists in the database
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 404)
+
+        # 404 when RFC document exists but has no StoredObject
+        WgRfcFactory(rfc_number=number)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 404)
+
+        # 404 when StoredObject exists but backing storage is missing (FileNotFoundError)
+        now = timezone.now()
+        StoredObject.objects.create(
+            store="rfc",
+            name=stored_name,
+            sha384="a" * 96,
+            len=0,
+            store_created=now,
+            created=now,
+            modified=now,
+        )
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 404)
+
+        # 200 with correct content-type, attachment disposition, and body when object is fully stored
+        xml_content = b"<rfc>test</rfc>"
+        store_bytes("rfc", stored_name, xml_content, allow_overwrite=True)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r["Content-Type"], "application/xml")
+        self.assertEqual(
+            r["Content-Disposition"],
+            f'attachment; filename="rfc{number}.notprepped.xml"',
+        )
+        self.assertEqual(b"".join(r.streaming_content), xml_content)
+
+    def test_rfcxml_notprepped_wrapper(self):
+        number = settings.FIRST_V3_RFC
+
+        # 404 for pre-v3 RFC numbers (no document needed)
+        r = self.client.get(
+            urlreverse(
+                "ietf.doc.views_doc.rfcxml_notprepped_wrapper",
+                kwargs=dict(number=number - 1),
+            )
+        )
+        self.assertEqual(r.status_code, 404)
+
+        # 404 when no RFC document exists in the database
+        r = self.client.get(
+            urlreverse(
+                "ietf.doc.views_doc.rfcxml_notprepped_wrapper",
+                kwargs=dict(number=number),
+            )
+        )
+        self.assertEqual(r.status_code, 404)
+
+        # 200 with rendered template when RFC document exists
+        rfc = WgRfcFactory(rfc_number=number)
+        r = self.client.get(
+            urlreverse(
+                "ietf.doc.views_doc.rfcxml_notprepped_wrapper",
+                kwargs=dict(number=number),
+            )
+        )
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        self.assertIn(str(rfc.rfc_number), q("h1").text())
+        download_url = urlreverse(
+            "ietf.doc.views_doc.rfcxml_notprepped", kwargs=dict(number=number)
+        )
+        self.assertEqual(len(q(f'a.btn[href="{download_url}"]')), 1)

ietf/doc/urls.py

@@ -99,6 +99,8 @@
 
     url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main),
     url(r'^%(name)s(?:/%(rev)s)?/bibtex/$' % settings.URL_REGEXPS, views_doc.document_bibtex),
+    url(r'^rfc(?P<number>[0-9]+)/notprepped/$' , views_doc.rfcxml_notprepped),
+    url(r'^rfc(?P<number>[0-9]+)/notprepped-wrapper/$', views_doc.rfcxml_notprepped_wrapper),
     url(r'^%(name)s(?:/%(rev)s)?/idnits2-state/$' % settings.URL_REGEXPS, views_doc.idnits2_state),
     url(r'^bibxml3/reference.I-D.%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml_ref),
     url(r'^bibxml3/%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml),

ietf/doc/views_doc.py

@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2009-2024, All Rights Reserved
+# Copyright The IETF Trust 2009-2026, All Rights Reserved
 # -*- coding: utf-8 -*-
 #
 # Parts Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies).
@@ -43,9 +43,10 @@
 
 from celery.result import AsyncResult
 from django.core.cache import caches
+from django.core.files.base import ContentFile
 from django.core.exceptions import PermissionDenied
 from django.db.models import Max
-from django.http import HttpResponse, Http404, HttpResponseBadRequest, JsonResponse
+from django.http import FileResponse, HttpResponse, Http404, HttpResponseBadRequest, JsonResponse
 from django.shortcuts import render, get_object_or_404, redirect
 from django.template.loader import render_to_string
 from django.urls import reverse as urlreverse
@@ -57,7 +58,7 @@
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import ( Document, DocHistory, DocEvent, BallotDocEvent, BallotType,
-    ConsensusDocEvent, NewRevisionDocEvent, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent,
+    ConsensusDocEvent, NewRevisionDocEvent, StoredObject, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent,
     IESG_BALLOT_ACTIVE_STATES, STATUSCHANGE_RELATIONS, DocumentActionHolder, DocumentAuthor,
     RelatedDocument, RelatedDocHistory)
 from ietf.doc.tasks import investigate_fragment_task
@@ -86,6 +87,7 @@
 from ietf.review.models import ReviewAssignment
 from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs, review_requests_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
+from ietf.doc.storage_utils import retrieve_bytes
 from ietf.utils import markup_txt, log, markdown
 from ietf.utils.draft import get_status_from_draft_text
 from ietf.utils.meetecho import MeetechoAPIError, SlidesManager
@@ -2356,3 +2358,29 @@ def investigate(request):
             "results": results,
         },
     )
+
+def rfcxml_notprepped(request, number):
+    number = int(number)
+    if number < settings.FIRST_V3_RFC:
+        raise Http404
+    rfc = Document.objects.filter(type="rfc", rfc_number=number).first()
+    if rfc is None:
+        raise Http404
+    name = f"notprepped/rfc{number}.notprepped.xml"
+    if not StoredObject.objects.filter(name=name).exists():
+        raise Http404
+    try:
+        bytes = retrieve_bytes("rfc", name)
+    except FileNotFoundError:
+        raise Http404
+    return FileResponse(ContentFile(bytes, name=f"rfc{number}.notprepped.xml"), as_attachment=True)
+
+
+def rfcxml_notprepped_wrapper(request, number):
+    number = int(number)
+    if number < settings.FIRST_V3_RFC:
+        raise Http404
+    rfc = Document.objects.filter(type="rfc", rfc_number=number).first()
+    if rfc is None:
+        raise Http404
+    return render(request, "doc/notprepped_wrapper.html", context={"rfc": rfc})