From d7d0e538c0ae8731c1ae043f6e5b9bb9cc5c94eb Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 16 Oct 2025 23:47:26 -0300 Subject: [PATCH 01/12] feat: ResolvedMaterial model + migration --- .../0017_resolvedmaterial_and_more.py | 48 +++++++++++++++++++ ietf/meeting/models.py | 21 ++++++++ 2 files changed, 69 insertions(+) create mode 100644 ietf/meeting/migrations/0017_resolvedmaterial_and_more.py diff --git a/ietf/meeting/migrations/0017_resolvedmaterial_and_more.py b/ietf/meeting/migrations/0017_resolvedmaterial_and_more.py new file mode 100644 index 0000000000..4a46c0c3ff --- /dev/null +++ b/ietf/meeting/migrations/0017_resolvedmaterial_and_more.py @@ -0,0 +1,48 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("meeting", "0016_alter_meeting_country_alter_meeting_time_zone"), + ] + + operations = [ + migrations.CreateModel( + name="ResolvedMaterial", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(help_text="Name to resolve", max_length=300)), + ( + "meeting_number", + models.CharField( + help_text="Meeting material is related to", max_length=64 + ), + ), + ( + "bucket", + models.CharField(help_text="Resolved bucket name", max_length=255), + ), + ( + "blob", + models.CharField(help_text="Resolved blob name", max_length=300), + ), + ], + ), + migrations.AddConstraint( + model_name="resolvedmaterial", + constraint=models.UniqueConstraint( + fields=("name", "meeting_number"), name="unique_name_per_meeting" + ), + ), + ] diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 9e44df33b7..c80544220b 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -956,6 +956,27 @@ class Meta: def __str__(self): return u"%s -> %s-%s" % (self.session, self.document.name, self.rev) + +class ResolvedMaterial(models.Model): + # A Document name can be 255 characters; allow this name to be a bit longer + name = models.CharField(max_length=300, help_text="Name to resolve") + meeting_number = models.CharField( + max_length=64, help_text="Meeting material is related to" + ) + bucket = models.CharField(max_length=255, help_text="Resolved bucket name") + blob = models.CharField(max_length=300, help_text="Resolved blob name") + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["name", "meeting_number"], name="unique_name_per_meeting" + ) + ] + + def __str__(self): + return f"{self.name}@{self.meeting_number} -> {self.bucket}:{self.blob}" + + constraint_cache_uses = 0 constraint_cache_initials = 0 From bbb3cd0ab75f050a48d980669b992f8132041327 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Oct 2025 11:35:24 -0300 Subject: [PATCH 02/12] feat: method to populate ResolvedMaterial (WIP) --- ietf/meeting/utils.py | 46 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index feadb0c7fd..893f73493d 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -17,7 +17,7 @@ from django.contrib import messages from django.core.cache import caches from django.core.files.base import ContentFile -from django.db import IntegrityError +from django.db import IntegrityError, transaction from django.db.models import OuterRef, Subquery, TextField, Q, Value, Max from django.db.models.functions import Coalesce from django.template.loader import render_to_string @@ -28,9 +28,19 @@ from ietf.dbtemplate.models import DBTemplate from ietf.doc.storage_utils import store_bytes, store_str -from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot, - Constraint, SchedTimeSessAssignment, SessionPresentation, Attended, - Registration, Meeting, RegistrationTicket) +from ietf.meeting.models import ( + Session, + SchedulingEvent, + TimeSlot, + Constraint, + SchedTimeSessAssignment, + SessionPresentation, + Attended, + Registration, + Meeting, + RegistrationTicket, + ResolvedMaterial, +) from ietf.doc.models import Document, State, NewRevisionDocEvent, StateDocEvent from ietf.doc.models import DocEvent from ietf.group.models import Group @@ -833,6 +843,34 @@ def write_doc_for_session(session, type_id, filename, contents): store_str(type_id, filename.name, contents) return None + +def resolve_materials_for_one_meeting(meeting: Meeting): + # todo think about whether this is safe enough against running trains + # Someone may update materials while we're gathering and we'll delete it in the + # transaction below. Is that a big enough risk to care? + meeting_documents = ( + Document.objects.exclude(type_id="draft").filter( + Q(session__meeting=meeting) | Q(proceedingsmaterial__meeting=meeting) + ) + ).distinct() + + resolved = [] + for doc in meeting_documents: + resolved.append( + ResolvedMaterial( + name=doc.name, + meeting_number=meeting.number, + bucket=doc.type_id, + blob=doc.get_base_name(), + ) + ) + # todo add lookups with revision in them + # todo check for existence of files / blobs + with transaction.atomic(): + ResolvedMaterial.objects.filter(meeting_number=meeting.number).delete() + ResolvedMaterial.objects.bulk_create(resolved) + + def create_recording(session, url, title=None, user=None): ''' Creates the Document type=recording, setting external_url and creating From 366bc6e97a703792ac1fd2f6e0ebeed20ea9e977 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Oct 2025 12:36:12 -0300 Subject: [PATCH 03/12] refactor: don't delete ResolvedMaterials Instead of deleting the ResolvedMaterials for a meeting, which might lose updates made during processing, update existing rows with any changes and warn if anything changed during the process. --- ietf/meeting/utils.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 893f73493d..601030fb4d 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -845,9 +845,7 @@ def write_doc_for_session(session, type_id, filename, contents): def resolve_materials_for_one_meeting(meeting: Meeting): - # todo think about whether this is safe enough against running trains - # Someone may update materials while we're gathering and we'll delete it in the - # transaction below. Is that a big enough risk to care? + start_time = timezone.now() meeting_documents = ( Document.objects.exclude(type_id="draft").filter( Q(session__meeting=meeting) | Q(proceedingsmaterial__meeting=meeting) @@ -866,10 +864,19 @@ def resolve_materials_for_one_meeting(meeting: Meeting): ) # todo add lookups with revision in them # todo check for existence of files / blobs - with transaction.atomic(): - ResolvedMaterial.objects.filter(meeting_number=meeting.number).delete() - ResolvedMaterial.objects.bulk_create(resolved) - + ResolvedMaterial.objects.bulk_create( + resolved, + update_conflicts=True, + unique_fields=["name", "meeting_number"], + update_fields=["bucket", "blob"], + ) + # Warn if any files were updated during the above process + last_update = meeting_documents.aggregate(Max("time"))["time__max"] + if last_update > start_time: + log( + f"Warning: materials for meeting {meeting.number} " + "changed during ResolvedMaterial update" + ) def create_recording(session, url, title=None, user=None): ''' From 89cff2208530803072cbc7a81470fff42466fdd2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Oct 2025 12:56:04 -0300 Subject: [PATCH 04/12] fix: fix _get_materials_doc() Did not handle the possibility of multiple DocHistory objects with the same rev. --- ietf/meeting/views.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index af3fe4b643..5dccb03888 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -281,10 +281,14 @@ def _matches_meeting(doc, meeting=None): docname, rev = name.rsplit("-", 1) if len(rev) == 2 and rev.isdigit(): try: - doc = DocHistory.objects.get(name=docname, rev=rev) - except DocHistory.DoesNotExist: # may raise Document.DoesNotExist doc = Document.objects.get(name=docname, rev=rev) + except Document.DoesNotExist: + doc = DocHistory.objects.filter( + name=docname, rev=rev, + ).order_by("-time").first() + if doc is None: + raise if ( _matches_meeting(doc, meeting) and rev in doc.revisions_by_newrevisionevent() From 5414662e470f8cb132c08e288c68a66bf5dbc1ef Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Oct 2025 16:11:05 -0300 Subject: [PATCH 05/12] refactor: factor out material lookup helper --- ietf/meeting/utils.py | 109 +++++++++++++++++++++++++++++++++++++++++- ietf/meeting/views.py | 92 +++-------------------------------- 2 files changed, 114 insertions(+), 87 deletions(-) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 601030fb4d..25b743a546 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- import datetime import itertools +from dataclasses import dataclass + import jsonschema import os import requests @@ -41,7 +43,14 @@ RegistrationTicket, ResolvedMaterial, ) -from ietf.doc.models import Document, State, NewRevisionDocEvent, StateDocEvent +from ietf.doc.models import ( + Document, + State, + NewRevisionDocEvent, + StateDocEvent, + DocHistory, + StoredObject, +) from ietf.doc.models import DocEvent from ietf.group.models import Group from ietf.group.utils import can_manage_materials @@ -843,6 +852,104 @@ def write_doc_for_session(session, type_id, filename, contents): store_str(type_id, filename.name, contents) return None +@dataclass +class BlobSpec: + bucket: str + name: str + + +def resolve_one_material( + doc: Document | DocHistory, rev: str | None, ext: str | None +) -> BlobSpec | None: + # Get the Document's base name. It may or may not have an extension. + if rev is None: + basename = Path(doc.get_base_name()) + else: + basename = Path(f"{doc.name}-{int(rev):02d}") + + # If we have an extension, either from the URL or the Document's base name, look up + # the blob or file or return 404. + if ext or basename.suffix != "": + if ext: + basename = basename.with_suffix(ext) + + # See if we have a stored object under that name + blob = ( + StoredObject.objects.exclude_deleted() + .filter(store=doc.type_id, name=basename) + .first() + ) + if blob is not None: + return BlobSpec( + bucket=blob.store, + name=blob.name, + ) + # No stored object, fall back to the file system. + filename = Path(doc.get_file_path()) / basename + if filename.exists(): + return BlobSpec( + bucket=doc.type_id, + name=str(basename), + ) + else: + return None + + # No extension has been specified so far, so look one up. + matching_stored_objects = ( + StoredObject.objects.exclude_deleted() + .filter( + store=doc.type_id, + name__startswith=f"{basename.stem}.", # anchor to end with trailing "." + ) + .order_by("name") + ) # orders by suffix + blob_ext_choices = { + Path(stored_obj.name).suffix: stored_obj + for stored_obj in matching_stored_objects + } + + # Short-circuit to return pdf if present + if ".pdf" in blob_ext_choices: + pdf_blob = blob_ext_choices[".pdf"] + return BlobSpec( + bucket=pdf_blob.store, + name=pdf_blob.name, + ) + + # Now look for files + filename = Path(doc.get_file_path()) / basename + file_ext_choices = { + # Construct a map from suffix to full filename + fn.suffix: fn.name + for fn in sorted(filename.parent.glob(filename.stem + ".*")) + } + + # Short-circuit to return pdf if we have the file + if ".pdf" in file_ext_choices: + pdf_filename = file_ext_choices[".pdf"] + return BlobSpec( + bucket=doc.type_id, + name=pdf_filename, + ) + + all_exts = set(blob_ext_choices.keys()).union(file_ext_choices.keys()) + if len(all_exts) > 0: + preferred_ext = sorted(all_exts)[0] + if preferred_ext in blob_ext_choices: + pdf_blob = blob_ext_choices[preferred_ext] + return BlobSpec( + bucket=pdf_blob.store, + name=pdf_blob.name, + ) + else: + pdf_filename = file_ext_choices[preferred_ext] + return BlobSpec( + bucket=doc.type_id, + name=pdf_filename, + ) + + return None + def resolve_materials_for_one_meeting(meeting: Meeting): start_time = timezone.now() diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 5dccb03888..55daaf71eb 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -52,13 +52,12 @@ from django.views.decorators.cache import cache_page from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt from django.views.generic import RedirectView -from rest_framework.status import HTTP_404_NOT_FOUND, HTTP_400_BAD_REQUEST +from rest_framework.status import HTTP_404_NOT_FOUND import debug # pyflakes:ignore from ietf.doc.fields import SearchableDocumentsField -from ietf.doc.models import Document, State, DocEvent, NewRevisionDocEvent, \ - StoredObject, DocHistory +from ietf.doc.models import Document, State, DocEvent, NewRevisionDocEvent, DocHistory from ietf.doc.storage_utils import ( remove_from_storage, retrieve_bytes, @@ -96,6 +95,7 @@ generate_proceedings_content, organize_proceedings_sessions, sort_accept_tuple, + resolve_one_material, ) from ietf.meeting.utils import add_event_info_to_session_qs from ietf.meeting.utils import session_time_for_sorting @@ -414,89 +414,9 @@ def _response(bucket: str, name: str): HTTP_404_NOT_FOUND, f"No such document for meeting {num}" ) - # Get the Document's base name. It may or may not have an extension. - if rev is None: - basename = Path(doc.get_base_name()) - else: - basename = Path(f"{doc.name}-{int(rev):02d}") - - # If we have an extension, either from the URL or the Document's base name, look up - # the blob or file or return 404. - if ext or basename.suffix != "": - if ext: - basename = basename.with_suffix(ext) - - # See if we have a stored object under that name - blob = StoredObject.objects.exclude_deleted().filter( - store=doc.type_id, name=basename - ).first() - if blob is not None: - return _response( - bucket=blob.store, - name=blob.name, - ) - # No stored object, fall back to the file system. - filename = Path(doc.get_file_path()) / basename - if filename.exists(): - return _response( - bucket=doc.type_id, - name=str(basename), - ) - else: - return _error_response( - HTTP_404_NOT_FOUND, - f"No file {basename} available for {document} for meeting {num}", - ) - - # No extension has been specified so far, so look one up. - matching_stored_objects = StoredObject.objects.exclude_deleted().filter( - store=doc.type_id, - name__startswith=f"{basename.stem}." # anchor to end with trailing "." - ).order_by("name") # orders by suffix - blob_ext_choices = { - Path(stored_obj.name).suffix: stored_obj - for stored_obj in matching_stored_objects - } - - # Short-circuit to return pdf if present - if ".pdf" in blob_ext_choices: - pdf_blob = blob_ext_choices[".pdf"] - return _response( - bucket=pdf_blob.store, - name=pdf_blob.name, - ) - - # Now look for files - filename = Path(doc.get_file_path()) / basename - file_ext_choices = { - # Construct a map from suffix to full filename - fn.suffix: fn.name - for fn in sorted(filename.parent.glob(filename.stem + ".*")) - } - - # Short-circuit to return pdf if we have the file - if ".pdf" in file_ext_choices: - pdf_filename = file_ext_choices[".pdf"] - return _response( - bucket=doc.type_id, - name=pdf_filename, - ) - - all_exts = set(blob_ext_choices.keys()).union(file_ext_choices.keys()) - if len(all_exts) > 0: - preferred_ext = sorted(all_exts)[0] - if preferred_ext in blob_ext_choices: - pdf_blob = blob_ext_choices[preferred_ext] - return _response( - bucket=pdf_blob.store, - name=pdf_blob.name, - ) - else: - pdf_filename = file_ext_choices[preferred_ext] - return _response( - bucket=doc.type_id, - name=pdf_filename, - ) + resolved = resolve_one_material(doc, rev, ext) + if resolved is not None: + return _response(bucket=resolved.bucket, name=resolved.name) return _error_response( HTTP_404_NOT_FOUND, f"No suitable file for {document} for meeting {num}" From dab91d320b408b7a452ffb323a8d95f3d6ffb669 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Oct 2025 16:24:59 -0300 Subject: [PATCH 06/12] feat: resolve blobs via blobdb/fs for cache --- ietf/meeting/utils.py | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 25b743a546..6cadaeed76 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -961,16 +961,45 @@ def resolve_materials_for_one_meeting(meeting: Meeting): resolved = [] for doc in meeting_documents: - resolved.append( + # request by doc name with no rev + blob = resolve_one_material(doc, rev=None, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=doc.name, + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + # request by doc name + rev + blob = resolve_one_material(doc, rev=doc.rev, ext=None) + if blob is not None: ResolvedMaterial( - name=doc.name, + name=f"{doc.name}-{doc.rev:02}", meeting_number=meeting.number, - bucket=doc.type_id, - blob=doc.get_base_name(), + bucket=blob.bucket, + blob=blob.name, ) - ) - # todo add lookups with revision in them - # todo check for existence of files / blobs + # for other revisions, only need request by doc name + rev + other_revisions = doc.revisions_by_newrevisionevent() + other_revisions.remove(doc.rev) + for rev in other_revisions: + old_doc = DocHistory.objects.filter( + doc=doc, rev=rev + ).order_by("-time").first() + if old_doc is None: + continue + blob = resolve_one_material(old_doc, rev=rev, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=f"{doc.name}-{rev:02}", + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) ResolvedMaterial.objects.bulk_create( resolved, update_conflicts=True, From 7839c1821a3e2724334aef38171ab700dff202ba Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Oct 2025 16:34:09 -0300 Subject: [PATCH 07/12] chore: add resource --- ietf/meeting/resources.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/ietf/meeting/resources.py b/ietf/meeting/resources.py index ede2b5b993..57be4487b6 100644 --- a/ietf/meeting/resources.py +++ b/ietf/meeting/resources.py @@ -11,11 +11,15 @@ from ietf import api -from ietf.meeting.models import ( Meeting, ResourceAssociation, Constraint, Room, Schedule, Session, - TimeSlot, SchedTimeSessAssignment, SessionPresentation, FloorPlan, - UrlResource, ImportantDate, SlideSubmission, SchedulingEvent, - BusinessConstraint, ProceedingsMaterial, MeetingHost, Attended, - Registration, RegistrationTicket) +from ietf.meeting.models import (Meeting, ResourceAssociation, Constraint, Room, + Schedule, Session, + TimeSlot, SchedTimeSessAssignment, SessionPresentation, + FloorPlan, + UrlResource, ImportantDate, SlideSubmission, + SchedulingEvent, + BusinessConstraint, ProceedingsMaterial, MeetingHost, + Attended, + Registration, RegistrationTicket, ResolvedMaterial) from ietf.name.resources import MeetingTypeNameResource class MeetingResource(ModelResource): @@ -472,3 +476,20 @@ class Meta: "registration": ALL_WITH_RELATIONS, } api.meeting.register(RegistrationTicketResource()) + + +class ResolvedMaterialResource(ModelResource): + class Meta: + queryset = ResolvedMaterial.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'resolvedmaterial' + ordering = ['id', ] + filtering = { + "id": ALL, + "name": ALL, + "meeting_number": ALL, + "bucket": ALL, + "blob": ALL, + } +api.meeting.register(ResolvedMaterialResource()) From 3344fd0dc432a419e0170af3cd441d05fba09995 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Oct 2025 16:41:14 -0300 Subject: [PATCH 08/12] feat: admin for ResolvedMaterial --- ietf/meeting/admin.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ietf/meeting/admin.py b/ietf/meeting/admin.py index d886a9a4b6..248838df18 100644 --- a/ietf/meeting/admin.py +++ b/ietf/meeting/admin.py @@ -9,7 +9,7 @@ SchedTimeSessAssignment, ResourceAssociation, FloorPlan, UrlResource, SessionPresentation, ImportantDate, SlideSubmission, SchedulingEvent, BusinessConstraint, ProceedingsMaterial, MeetingHost, Registration, RegistrationTicket, - AttendanceTypeName) + AttendanceTypeName, ResolvedMaterial) class UrlResourceAdmin(admin.ModelAdmin): @@ -288,3 +288,13 @@ def display_meeting(self, instance): display_meeting.short_description = "Meeting" # type: ignore # https://github.com/python/mypy/issues/2087 admin.site.register(RegistrationTicket, RegistrationTicketAdmin) + + +class ResolvedMaterialAdmin(admin.ModelAdmin): + model = ResolvedMaterial + list_display = ["name", "meeting_number", "bucket", "blob"] + list_filter = ["meeting_number", "bucket"] + search_fields = ["name", "blob"] + ordering = ["name"] + +admin.site.register(ResolvedMaterial, ResolvedMaterialAdmin) From 336324d46f5125825a71e96e1f7680012e8ddf55 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Oct 2025 16:54:40 -0300 Subject: [PATCH 09/12] feat: cache-driven resolve materials API --- ietf/api/urls.py | 1 + ietf/meeting/views.py | 53 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/ietf/api/urls.py b/ietf/api/urls.py index b6bf29588e..6c9740f115 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -51,6 +51,7 @@ url(r'^iesg/position', views_ballot.api_set_position), # Find the blob to store for a given materials document path url(r'^meeting/(?:(?P(?:interim-)?[a-z0-9-]+)/)?materials/%(document)s(?P\.[A-Za-z0-9]+)?/resolve/$' % settings.URL_REGEXPS, meeting_views.api_resolve_materials_name), + url(r'^meeting/(?:(?P(?:interim-)?[a-z0-9-]+)/)?materials/%(document)s(?P\.[A-Za-z0-9]+)?/resolve-cached/$' % settings.URL_REGEXPS, meeting_views.api_resolve_materials_name_cached), url(r'^meeting/blob/(?P[a-z0-9-]+)/(?P[a-z][a-z0-9.-]+)$', meeting_views.api_retrieve_materials_blob), # Let Meetecho set session video URLs url(r'^meeting/session/video/url$', meeting_views.api_set_session_video_url), diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 55daaf71eb..271f66755a 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -70,7 +70,8 @@ from ietf.person.models import Person, User from ietf.ietfauth.utils import role_required, has_role, user_is_person from ietf.mailtrigger.utils import gather_address_lists -from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, SessionPresentation, TimeSlot, SlideSubmission, Attended +from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, \ + SessionPresentation, TimeSlot, SlideSubmission, Attended, ResolvedMaterial from ietf.meeting.models import ImportantDate, SessionStatusName, SchedulingEvent, SchedTimeSessAssignment, Room, TimeSlotTypeName from ietf.meeting.models import Registration from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm, ImportMinutesForm, @@ -371,16 +372,17 @@ def materials_document(request, document, num=None, ext=None): @requires_api_token def api_resolve_materials_name(request, document, num=None, ext=None): """Resolve materials name into document to a blob spec - + Returns the bucket/name of a blob in the blob store that corresponds to the named document. Handles resolution of revision if it is not specified and determines the best extension if one is not provided. Response is JSON. - + As of 2025-10-10 we do not have blobs for all materials documents or for every format of every document. This API still returns the bucket/name as if the blob exists. Another API will allow the caller to obtain the file contents using that name if it cannot be retrieved from the blob store. """ + def _error_response(status: int, detail: str): return JsonResponse( { @@ -390,7 +392,7 @@ def _error_response(status: int, detail: str): }, status=status, ) - + def _response(bucket: str, name: str): return JsonResponse( { @@ -423,6 +425,49 @@ def _response(bucket: str, name: str): ) +@requires_api_token("ietf.meeting.views.api_resolve_materials_name") +def api_resolve_materials_name_cached(request, document, num=None, ext=None): + """Resolve materials name into document to a blob spec + + Returns the bucket/name of a blob in the blob store that corresponds to the named + document. Handles resolution of revision if it is not specified and determines the + best extension if one is not provided. Response is JSON. + + As of 2025-10-10 we do not have blobs for all materials documents or for every + format of every document. This API still returns the bucket/name as if the blob + exists. Another API will allow the caller to obtain the file contents using that + name if it cannot be retrieved from the blob store. + """ + + def _error_response(status: int, detail: str): + return JsonResponse( + { + "status": status, + "title": "Error", + "detail": detail, + }, + status=status, + ) + + def _response(bucket: str, name: str): + return JsonResponse( + { + "bucket": bucket, + "name": name, + } + ) + + try: + resolved = ResolvedMaterial.objects.get( + meeting_number=num, name=document + ) + except ResolvedMaterial.DoesNotExist: + return _error_response( + HTTP_404_NOT_FOUND, f"No suitable file for {document} for meeting {num}" + ) + return _response(bucket=resolved.bucket, name=resolved.blob) + + @requires_api_token def api_retrieve_materials_blob(request, bucket, name): """Retrieve contents of a meeting materials blob From 9c059ee6f089bbda5ecfc2911f7696b2ccb13379 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Oct 2025 17:13:07 -0300 Subject: [PATCH 10/12] fix: add all ResolvedMaterials; var names --- ietf/meeting/utils.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 6cadaeed76..73e16939ec 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -852,6 +852,7 @@ def write_doc_for_session(session, type_id, filename, contents): store_str(type_id, filename.name, contents) return None + @dataclass class BlobSpec: bucket: str @@ -874,15 +875,15 @@ def resolve_one_material( basename = basename.with_suffix(ext) # See if we have a stored object under that name - blob = ( + preferred_blob = ( StoredObject.objects.exclude_deleted() .filter(store=doc.type_id, name=basename) .first() ) - if blob is not None: + if preferred_blob is not None: return BlobSpec( - bucket=blob.store, - name=blob.name, + bucket=preferred_blob.store, + name=preferred_blob.name, ) # No stored object, fall back to the file system. filename = Path(doc.get_file_path()) / basename @@ -936,16 +937,16 @@ def resolve_one_material( if len(all_exts) > 0: preferred_ext = sorted(all_exts)[0] if preferred_ext in blob_ext_choices: - pdf_blob = blob_ext_choices[preferred_ext] + preferred_blob = blob_ext_choices[preferred_ext] return BlobSpec( - bucket=pdf_blob.store, - name=pdf_blob.name, + bucket=preferred_blob.store, + name=preferred_blob.name, ) else: - pdf_filename = file_ext_choices[preferred_ext] + preferred_filename = file_ext_choices[preferred_ext] return BlobSpec( bucket=doc.type_id, - name=pdf_filename, + name=preferred_filename, ) return None @@ -975,11 +976,13 @@ def resolve_materials_for_one_meeting(meeting: Meeting): # request by doc name + rev blob = resolve_one_material(doc, rev=doc.rev, ext=None) if blob is not None: - ResolvedMaterial( - name=f"{doc.name}-{doc.rev:02}", - meeting_number=meeting.number, - bucket=blob.bucket, - blob=blob.name, + resolved.append( + ResolvedMaterial( + name=f"{doc.name}-{doc.rev:02}", + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) ) # for other revisions, only need request by doc name + rev other_revisions = doc.revisions_by_newrevisionevent() From cad23e1058ac63544f1a0312552812c05733e5d6 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Oct 2025 17:38:43 -0300 Subject: [PATCH 11/12] fix: handle null case --- ietf/meeting/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 73e16939ec..1d9da72fce 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -1011,7 +1011,7 @@ def resolve_materials_for_one_meeting(meeting: Meeting): ) # Warn if any files were updated during the above process last_update = meeting_documents.aggregate(Max("time"))["time__max"] - if last_update > start_time: + if last_update and last_update > start_time: log( f"Warning: materials for meeting {meeting.number} " "changed during ResolvedMaterial update" From 0ed88b030e05b6b8e79e9eae660c270b12c032b2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Oct 2025 17:39:09 -0300 Subject: [PATCH 12/12] feat: resolve_meeting_materials_task --- ietf/meeting/tasks.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py index 784eb00d87..d35f78979d 100644 --- a/ietf/meeting/tasks.py +++ b/ietf/meeting/tasks.py @@ -2,12 +2,14 @@ # # Celery task definitions # +import datetime + from celery import shared_task from django.utils import timezone from ietf.utils import log from .models import Meeting -from .utils import generate_proceedings_content +from .utils import generate_proceedings_content, resolve_materials_for_one_meeting from .views import generate_agenda_data from .utils import fetch_attendance_from_meetings @@ -61,3 +63,24 @@ def fetch_meeting_attendance_task(): meeting_stats['processed'] ) ) + + +@shared_task +def resolve_meeting_materials_task(*, meetings=None, meetings_since=None): + if meetings_since is not None: + meetings_since = datetime.datetime.fromisoformat(meetings_since) + if meetings is None: + if meetings_since is None: + log.log("No meetings requested, doing nothing.") + return + meetings = Meeting.objects.filter(date__gte=meetings_since) + log.log(f"Resolving materials for meetings since {meetings_since}") + else: + if meetings_since is not None: + log.log("Ignoring meetings_since because specific meetings were requested.") + meetings = Meeting.objects.filter(number__in=meetings) + for meeting in meetings: + log.log(f"Resolving materials for {meeting.type_id} meeting {meeting.number}...") + mark = timezone.now() + resolve_materials_for_one_meeting(meeting) + log.log(f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds.")