From dffc60ae9eafdce5fe68096df55d50c74eee9559 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 18 Sep 2025 11:47:47 -0300 Subject: [PATCH 1/8] refactor: central def of subseries doc types --- ietf/doc/models.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 52a42e845cc..20f500d4495 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -937,7 +937,18 @@ def role_for_doc(self): 'invalid' ) + +SUBSERIES_DOC_TYPE_IDS = ("bcp", "fyi", "std") + + +class DocumentQuerySet(models.QuerySet): + def subseries_docs(self): + return self.filter(type_id__in=SUBSERIES_DOC_TYPE_IDS) + + class Document(StorableMixin, DocumentInfo): + objects = DocumentQuerySet.as_manager() + name = models.CharField(max_length=255, validators=[validate_docname,], unique=True) # immutable action_holders = models.ManyToManyField(Person, through=DocumentActionHolder, blank=True) From 0f6655626646389497a16d54f491cf157e5a45a4 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 18 Sep 2025 12:06:53 -0300 Subject: [PATCH 2/8] feat: subseries doc API --- ietf/api/urls.py | 1 + ietf/doc/api.py | 74 +++++++++++++++++++++++++++++++++++------ ietf/doc/serializers.py | 37 ++++++++++++++++++++- 3 files changed, 100 insertions(+), 12 deletions(-) diff --git a/ietf/api/urls.py b/ietf/api/urls.py index a88ea5d6629..4e4cf4be681 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -24,6 +24,7 @@ # todo more general name for this API? red_router = PrefixedSimpleRouter(name_prefix="ietf.api.red_api") # red api router red_router.register("doc", doc_api.RfcViewSet) +red_router.register("subseries", doc_api.SubseriesViewSet, basename="subseries") api.autodiscover() diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 2fecdfa2394..2a6cb81ced9 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2024, All Rights Reserved """Doc API implementations""" -from django.db.models import OuterRef, Subquery, Prefetch, Value, JSONField + +from django.db.models import OuterRef, Subquery, Prefetch, Value, JSONField, QuerySet from django.db.models.functions import TruncDate from django_filters import rest_framework as filters from rest_framework import filters as drf_filters @@ -13,7 +14,12 @@ from ietf.name.models import StreamName from ietf.utils.timezone import RPC_TZINFO from .models import Document, DocEvent, RelatedDocument -from .serializers import RfcMetadataSerializer, RfcStatus, RfcSerializer +from .serializers import ( + RfcMetadataSerializer, + RfcStatus, + RfcSerializer, + SubseriesDocSerializer, +) class RfcLimitOffsetPagination(LimitOffsetPagination): @@ -64,17 +70,14 @@ def __init__(self, to_attr, relationship_id, reverse=False, doc_type_id="rfc"): "relationship_id": relationship_id, f"{'source' if reverse else 'target'}__type_id": doc_type_id, } - ), + ).select_related("source" if reverse else "target"), to_attr=to_attr, ) -class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): - permission_classes: list[BasePermission] = [] - lookup_field = "rfc_number" - queryset = ( - Document.objects.filter(type_id="rfc", rfc_number__isnull=False) - .annotate( +def augment_rfc_queryset(queryset: QuerySet[Document]): + return ( + queryset.annotate( published_datetime=Subquery( DocEvent.objects.filter( doc_id=OuterRef("pk"), @@ -85,7 +88,6 @@ class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): ), ) .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) - .order_by("-rfc_number") .prefetch_related( PrefetchRelatedDocument( to_attr="drafts", @@ -110,7 +112,16 @@ class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): keywords=Value(["keyword"], output_field=JSONField()), errata=Value([], output_field=JSONField()), ) - ) # default ordering - RfcFilter may override + ) + + +class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + permission_classes: list[BasePermission] = [] + lookup_field = "rfc_number" + queryset = augment_rfc_queryset( + Document.objects.filter(type_id="rfc", rfc_number__isnull=False) + ).order_by("-rfc_number") + pagination_class = RfcLimitOffsetPagination filter_backends = [filters.DjangoFilterBackend, drf_filters.SearchFilter] filterset_class = RfcFilter @@ -120,3 +131,44 @@ def get_serializer_class(self): if self.action == "retrieve": return RfcSerializer return RfcMetadataSerializer + + +class PrefetchSubseriesContents(Prefetch): + # Queryset used for the prefetch + _queryset = augment_rfc_queryset( + Document.objects.annotate( + published_datetime=Subquery( + DocEvent.objects.filter( + doc_id=OuterRef("pk"), + type="published_rfc", + ) + .order_by("-time") + .values("time")[:1] + ), + ) + ) + + def __init__(self, to_attr): + super().__init__( + lookup="relateddocument_set", + queryset=RelatedDocument.objects.filter( + relationship_id="contains", + target__type_id="rfc", + ).prefetch_related( + Prefetch( + "target", + queryset=self._queryset, + to_attr="document", + ) + ), + to_attr=to_attr, + ) + + +class SubseriesViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + permission_classes: list[BasePermission] = [] + lookup_field = "name" + serializer_class = SubseriesDocSerializer + queryset = Document.objects.subseries_docs().prefetch_related( + PrefetchSubseriesContents(to_attr="contents") + ) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index ff590e8700d..107404e5a93 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -3,12 +3,13 @@ from dataclasses import dataclass from typing import Literal, ClassVar +from django.db.models.manager import BaseManager from drf_spectacular.utils import extend_schema_field from rest_framework import serializers, fields from ietf.group.serializers import GroupSerializer from ietf.name.serializers import StreamNameSerializer -from .models import Document, DocumentAuthor +from .models import Document, DocumentAuthor, RelatedDocument class RfcAuthorSerializer(serializers.ModelSerializer): @@ -199,3 +200,37 @@ class RfcSerializer(RfcMetadataSerializer): class Meta: model = RfcMetadataSerializer.Meta.model fields = RfcMetadataSerializer.Meta.fields + ["text"] + + +class SubseriesContentListSerializer(serializers.ListSerializer): + """ListSerializer that gets its object from item.document""" + + def to_representation(self, data): + """ + List of object instances -> List of dicts of primitive datatypes. + """ + # Dealing with nested relationships, data can be a Manager, + # so, first get a queryset from the Manager if needed + iterable = ( + data.all() if isinstance(data, BaseManager) else data + ) + # Serialize item.document instead of item itself + return [self.child.to_representation(item.document) for item in iterable] + + +class SubseriesContentSerializer(RfcMetadataSerializer): + class Meta(RfcMetadataSerializer.Meta): + list_serializer_class = SubseriesContentListSerializer + + +class SubseriesDocSerializer(serializers.ModelSerializer): + """Serialize a subseries document (e.g., a BCP or STD)""" + contents = SubseriesContentSerializer(many=True) + + class Meta: + model = Document + fields = [ + "name", + "type", + "contents", + ] From ef6906c286086c0df20921e7c2ad4010bc93ac2a Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 18 Sep 2025 13:14:06 -0300 Subject: [PATCH 3/8] refactor: optimize queries via prefetch Reduced 4500 to 18 queries --- ietf/doc/api.py | 82 ++++++++++++++++++++++++----------------- ietf/doc/serializers.py | 26 ++++++++++++- 2 files changed, 73 insertions(+), 35 deletions(-) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 2a6cb81ced9..a9d38e55678 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -13,7 +13,7 @@ from ietf.group.models import Group from ietf.name.models import StreamName from ietf.utils.timezone import RPC_TZINFO -from .models import Document, DocEvent, RelatedDocument +from .models import Document, DocEvent, RelatedDocument, DocumentAuthor from .serializers import ( RfcMetadataSerializer, RfcStatus, @@ -61,34 +61,37 @@ class PrefetchRelatedDocument(Prefetch): those for which the current RFC is the `source`. If `reverse` is True, includes those for which it is the `target` instead. Defaults to only "rfc" documents. """ + @staticmethod + def _get_queryset(relationship_id, reverse, doc_type_id): + """Get queryset to use for the prefetch""" + return RelatedDocument.objects.filter( + **{ + "relationship_id": relationship_id, + f"{'source' if reverse else 'target'}__type_id": doc_type_id, + } + ).select_related("source" if reverse else "target") def __init__(self, to_attr, relationship_id, reverse=False, doc_type_id="rfc"): super().__init__( lookup="targets_related" if reverse else "relateddocument_set", - queryset=RelatedDocument.objects.filter( - **{ - "relationship_id": relationship_id, - f"{'source' if reverse else 'target'}__type_id": doc_type_id, - } - ).select_related("source" if reverse else "target"), + queryset=self._get_queryset(relationship_id, reverse, doc_type_id), to_attr=to_attr, ) def augment_rfc_queryset(queryset: QuerySet[Document]): return ( - queryset.annotate( - published_datetime=Subquery( - DocEvent.objects.filter( - doc_id=OuterRef("pk"), - type="published_rfc", - ) - .order_by("-time") - .values("time")[:1] - ), - ) - .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) + queryset + .select_related("std_level", "stream") .prefetch_related( + Prefetch( + "group", + Group.objects.select_related("parent"), + ), + Prefetch( + "documentauthor_set", + DocumentAuthor.objects.select_related("email", "person"), + ), PrefetchRelatedDocument( to_attr="drafts", relationship_id="became_rfc", @@ -104,6 +107,17 @@ def augment_rfc_queryset(queryset: QuerySet[Document]): to_attr="updated_by", relationship_id="updates", reverse=True ), ) + .annotate( + published_datetime=Subquery( + DocEvent.objects.filter( + doc_id=OuterRef("pk"), + type="published_rfc", + ) + .order_by("-time") + .values("time")[:1] + ), + ) + .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) .annotate( # TODO implement these fake fields for real is_also=Value([], output_field=JSONField()), @@ -134,20 +148,6 @@ def get_serializer_class(self): class PrefetchSubseriesContents(Prefetch): - # Queryset used for the prefetch - _queryset = augment_rfc_queryset( - Document.objects.annotate( - published_datetime=Subquery( - DocEvent.objects.filter( - doc_id=OuterRef("pk"), - type="published_rfc", - ) - .order_by("-time") - .values("time")[:1] - ), - ) - ) - def __init__(self, to_attr): super().__init__( lookup="relateddocument_set", @@ -157,7 +157,7 @@ def __init__(self, to_attr): ).prefetch_related( Prefetch( "target", - queryset=self._queryset, + queryset=augment_rfc_queryset(Document.objects.all()), to_attr="document", ) ), @@ -172,3 +172,19 @@ class SubseriesViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): queryset = Document.objects.subseries_docs().prefetch_related( PrefetchSubseriesContents(to_attr="contents") ) + + def list(self, request, *args, **kwargs): + from django.db import connection, reset_queries + reset_queries() + result = super().list(request, *args, **kwargs) + print("\n\n".join(q["sql"] for q in connection.queries)) + print(f"\n\nTotal: {len(connection.queries)} queries") + return result + + def retrieve(self, request, *args, **kwargs): + from django.db import connection, reset_queries + reset_queries() + result = super().retrieve(request, *args, **kwargs) + print("\n\n".join(q["sql"] for q in connection.queries)) + print(f"\n\nTotal: {len(connection.queries)} queries") + return result diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index 107404e5a93..f82961367c1 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -145,7 +145,7 @@ class RfcMetadataSerializer(serializers.ModelSerializer): area = GroupSerializer(source="group.area", required=False) stream = StreamNameSerializer() identifiers = fields.SerializerMethodField() - draft = RelatedDraftSerializer(source="came_from_draft", read_only=True) # todo prefetch this + draft = RelatedDraftSerializer(source="drafts.first", read_only=True) obsoletes = RelatedRfcSerializer(many=True, read_only=True) obsoleted_by = ReverseRelatedRfcSerializer(many=True, read_only=True) updates = RelatedRfcSerializer(many=True, read_only=True) @@ -221,7 +221,29 @@ def to_representation(self, data): class SubseriesContentSerializer(RfcMetadataSerializer): class Meta(RfcMetadataSerializer.Meta): list_serializer_class = SubseriesContentListSerializer - + fields = [ + "number", + "title", + "published", + "status", + "pages", + "authors", + "group", + "area", + "stream", + "identifiers", + "obsoletes", + "obsoleted_by", + "updates", + "updated_by", + "is_also", + "see_also", + "draft", + "abstract", + "formats", + "keywords", + "errata", + ] class SubseriesDocSerializer(serializers.ModelSerializer): """Serialize a subseries document (e.g., a BCP or STD)""" From 9a6cdfcd5c4d5c54854168e2860540a2f43c66e8 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 18 Sep 2025 13:14:25 -0300 Subject: [PATCH 4/8] chore: remove debug --- ietf/doc/api.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index a9d38e55678..eac86b38103 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -172,19 +172,3 @@ class SubseriesViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): queryset = Document.objects.subseries_docs().prefetch_related( PrefetchSubseriesContents(to_attr="contents") ) - - def list(self, request, *args, **kwargs): - from django.db import connection, reset_queries - reset_queries() - result = super().list(request, *args, **kwargs) - print("\n\n".join(q["sql"] for q in connection.queries)) - print(f"\n\nTotal: {len(connection.queries)} queries") - return result - - def retrieve(self, request, *args, **kwargs): - from django.db import connection, reset_queries - reset_queries() - result = super().retrieve(request, *args, **kwargs) - print("\n\n".join(q["sql"] for q in connection.queries)) - print(f"\n\nTotal: {len(connection.queries)} queries") - return result From 677693198c6a82f389a28e2b0d515530412aaeab Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 18 Sep 2025 14:51:36 -0300 Subject: [PATCH 5/8] fix: fix serialization of draft field --- ietf/doc/serializers.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index f82961367c1..c7e917275cf 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -145,7 +145,7 @@ class RfcMetadataSerializer(serializers.ModelSerializer): area = GroupSerializer(source="group.area", required=False) stream = StreamNameSerializer() identifiers = fields.SerializerMethodField() - draft = RelatedDraftSerializer(source="drafts.first", read_only=True) + draft = serializers.SerializerMethodField() obsoletes = RelatedRfcSerializer(many=True, read_only=True) obsoleted_by = ReverseRelatedRfcSerializer(many=True, read_only=True) updates = RelatedRfcSerializer(many=True, read_only=True) @@ -191,6 +191,14 @@ def get_identifiers(self, doc: Document): ) return DocIdentifierSerializer(instance=identifiers, many=True).data + @extend_schema_field(RelatedDraftSerializer) + def get_draft(self, object): + try: + related_doc = object.drafts[0] + except IndexError: + return None + return RelatedDraftSerializer(related_doc.source).data + class RfcSerializer(RfcMetadataSerializer): """Serialize an RFC, including its metadata and text content if available""" From bc5a4769a03b7728804d134ec2e5f3c82279ea52 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 18 Sep 2025 17:52:22 -0300 Subject: [PATCH 6/8] refactor: clean up prefetch a bit --- ietf/doc/api.py | 1 - ietf/doc/serializers.py | 41 ++++++++++------------------------------- 2 files changed, 10 insertions(+), 32 deletions(-) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index eac86b38103..540820759e7 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -158,7 +158,6 @@ def __init__(self, to_attr): Prefetch( "target", queryset=augment_rfc_queryset(Document.objects.all()), - to_attr="document", ) ), to_attr=to_attr, diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index c7e917275cf..4e130966f7d 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -115,10 +115,10 @@ def to_representation(self, instance: Document): return super().to_representation(instance=RfcStatus.from_document(instance)) -class RelatedDraftSerializer(serializers.ModelSerializer): - class Meta: - model = Document - fields = ["id", "name", "title"] +class RelatedDraftSerializer(serializers.Serializer): + id = serializers.IntegerField(source="source.id") + name = serializers.CharField(source="source.name") + title = serializers.CharField(source="source.title") class RelatedRfcSerializer(serializers.Serializer): @@ -197,7 +197,7 @@ def get_draft(self, object): related_doc = object.drafts[0] except IndexError: return None - return RelatedDraftSerializer(related_doc.source).data + return RelatedDraftSerializer(related_doc).data class RfcSerializer(RfcMetadataSerializer): @@ -211,7 +211,7 @@ class Meta: class SubseriesContentListSerializer(serializers.ListSerializer): - """ListSerializer that gets its object from item.document""" + """ListSerializer that gets its object from item.target""" def to_representation(self, data): """ @@ -222,36 +222,15 @@ def to_representation(self, data): iterable = ( data.all() if isinstance(data, BaseManager) else data ) - # Serialize item.document instead of item itself - return [self.child.to_representation(item.document) for item in iterable] + # Serialize item.target instead of item itself + return [self.child.to_representation(item.target) for item in iterable] class SubseriesContentSerializer(RfcMetadataSerializer): + """Serialize RFC contained in a subseries doc""" class Meta(RfcMetadataSerializer.Meta): list_serializer_class = SubseriesContentListSerializer - fields = [ - "number", - "title", - "published", - "status", - "pages", - "authors", - "group", - "area", - "stream", - "identifiers", - "obsoletes", - "obsoleted_by", - "updates", - "updated_by", - "is_also", - "see_also", - "draft", - "abstract", - "formats", - "keywords", - "errata", - ] + class SubseriesDocSerializer(serializers.ModelSerializer): """Serialize a subseries document (e.g., a BCP or STD)""" From 0d3d913d542405f465a57bd9b8118ef77ff179d1 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 18 Sep 2025 18:06:29 -0300 Subject: [PATCH 7/8] feat: filter by subseries type --- ietf/doc/api.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 540820759e7..579cdefaf0e 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -11,9 +11,10 @@ from rest_framework.viewsets import GenericViewSet from ietf.group.models import Group -from ietf.name.models import StreamName +from ietf.name.models import StreamName, DocTypeName from ietf.utils.timezone import RPC_TZINFO -from .models import Document, DocEvent, RelatedDocument, DocumentAuthor +from .models import Document, DocEvent, RelatedDocument, DocumentAuthor, \ + SUBSERIES_DOC_TYPE_IDS from .serializers import ( RfcMetadataSerializer, RfcStatus, @@ -24,7 +25,7 @@ class RfcLimitOffsetPagination(LimitOffsetPagination): default_limit = 10 - max_limit = 500 + max_limit = 50000 class RfcFilter(filters.FilterSet): @@ -164,6 +165,12 @@ def __init__(self, to_attr): ) +class SubseriesFilter(filters.FilterSet): + type = filters.ModelMultipleChoiceFilter( + queryset=DocTypeName.objects.filter(pk__in=SUBSERIES_DOC_TYPE_IDS) + ) + + class SubseriesViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): permission_classes: list[BasePermission] = [] lookup_field = "name" @@ -171,3 +178,5 @@ class SubseriesViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): queryset = Document.objects.subseries_docs().prefetch_related( PrefetchSubseriesContents(to_attr="contents") ) + filter_backends = [filters.DjangoFilterBackend] + filterset_class = SubseriesFilter From c259c0c81953c0844548acbafb38f9a7560b66c4 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 18 Sep 2025 18:25:44 -0300 Subject: [PATCH 8/8] fix: restore max_limit for RFC pagination --- ietf/doc/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 579cdefaf0e..81e9df49025 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -25,7 +25,7 @@ class RfcLimitOffsetPagination(LimitOffsetPagination): default_limit = 10 - max_limit = 50000 + max_limit = 500 class RfcFilter(filters.FilterSet):