feat: blobstore-driven meeting materials

ietf/api/urls.py

@@ -49,6 +49,9 @@
     url(r'^group/role-holder-addresses/$', api_views.role_holder_addresses),
     # Let IESG members set positions programmatically
     url(r'^iesg/position', views_ballot.api_set_position),
+    # Find the blob to store for a given materials document path
+    url(r'^meeting/(?:(?P<num>(?:interim-)?[a-z0-9-]+)/)?materials/%(document)s(?P<ext>\.[A-Za-z0-9]+)?/resolve-cached/$' % settings.URL_REGEXPS, meeting_views.api_resolve_materials_name_cached),
+    url(r'^meeting/blob/(?P<bucket>[a-z0-9-]+)/(?P<name>[a-z][a-z0-9.-]+)$', meeting_views.api_retrieve_materials_blob),
     # Let Meetecho set session video URLs
     url(r'^meeting/session/video/url$', meeting_views.api_set_session_video_url),
     # Let Meetecho tell us the name of its recordings

ietf/blobdb/admin.py

@@ -3,7 +3,7 @@
 from django.db.models.functions import Length
 from rangefilter.filters import DateRangeQuickSelectListFilterBuilder
 
-from .models import Blob
+from .models import Blob, ResolvedMaterial
 
 
 @admin.register(Blob)
@@ -29,3 +29,12 @@ def get_queryset(self, request):
     def object_size(self, instance):
         """Get the size of the object"""
         return instance.object_size  # annotation added in get_queryset()
+
+
+@admin.register(ResolvedMaterial)
+class ResolvedMaterialAdmin(admin.ModelAdmin):
+    model = ResolvedMaterial
+    list_display = ["name", "meeting_number", "bucket", "blob"]
+    list_filter = ["meeting_number", "bucket"]
+    search_fields = ["name", "blob"]
+    ordering = ["name"]

ietf/blobdb/migrations/0002_resolvedmaterial.py

@@ -0,0 +1,48 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("blobdb", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ResolvedMaterial",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(help_text="Name to resolve", max_length=300)),
+                (
+                    "meeting_number",
+                    models.CharField(
+                        help_text="Meeting material is related to", max_length=64
+                    ),
+                ),
+                (
+                    "bucket",
+                    models.CharField(help_text="Resolved bucket name", max_length=255),
+                ),
+                (
+                    "blob",
+                    models.CharField(help_text="Resolved blob name", max_length=300),
+                ),
+            ],
+        ),
+        migrations.AddConstraint(
+            model_name="resolvedmaterial",
+            constraint=models.UniqueConstraint(
+                fields=("name", "meeting_number"), name="unique_name_per_meeting"
+            ),
+        ),
+    ]

ietf/blobdb/models.py

@@ -96,3 +96,23 @@ def _emit_blob_change_event(self, using=None):
             ),
             using=using,
         )
+
+
+class ResolvedMaterial(models.Model):
+    # A Document name can be 255 characters; allow this name to be a bit longer
+    name = models.CharField(max_length=300, help_text="Name to resolve")
+    meeting_number = models.CharField(
+        max_length=64, help_text="Meeting material is related to"
+    )
+    bucket = models.CharField(max_length=255, help_text="Resolved bucket name")
+    blob = models.CharField(max_length=300, help_text="Resolved blob name")
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["name", "meeting_number"], name="unique_name_per_meeting"
+            )
+        ]
+
+    def __str__(self):
+        return f"{self.name}@{self.meeting_number} -> {self.bucket}:{self.blob}"

ietf/doc/models.py

@@ -913,6 +913,7 @@ def role_for_doc(self):
             roles.append('Action Holder')
         return ', '.join(roles) 
 
+# N.B., at least a couple dozen documents exist that do not satisfy this validator
 validate_docname = RegexValidator(
     r'^[-a-z0-9]+$',
     "Provide a valid document name consisting of lowercase letters, numbers and hyphens.",
@@ -1588,9 +1589,17 @@ class BofreqResponsibleDocEvent(DocEvent):
     """ Capture the responsible leadership (IAB and IESG members) for a BOF Request """
     responsible = models.ManyToManyField('person.Person', blank=True)
 
+
+class StoredObjectQuerySet(models.QuerySet):
+    def exclude_deleted(self):
+        return self.filter(deleted__isnull=True)
+
+
 class StoredObject(models.Model):
     """Hold metadata about objects placed in object storage"""
 
+    objects = StoredObjectQuerySet.as_manager() 
+
     store = models.CharField(max_length=256)
     name = models.CharField(max_length=1024, null=False, blank=False) # N.B. the 1024 limit on name comes from S3
     sha384 = models.CharField(max_length=96)

ietf/doc/storage.py

@@ -32,7 +32,7 @@ def __init__(self, file, name, mtime=None, content_type="", store=None, doc_name
     @classmethod
     def from_storedobject(cls, file, name, store):
         """Alternate constructor for objects that already exist in the StoredObject table"""
-        stored_object = StoredObject.objects.filter(store=store, name=name, deleted__isnull=True).first()
+        stored_object = StoredObject.objects.exclude_deleted().filter(store=store, name=name).first()
         if stored_object is None:
             raise FileNotFoundError(f"StoredObject for {store}:{name} does not exist or was deleted")
         file = cls(file, name, store, doc_name=stored_object.doc_name, doc_rev=stored_object.doc_rev)
@@ -140,7 +140,11 @@ def _save_stored_object(self, name, content) -> StoredObject:
                 ),
             ),
         )
-        if not created:
+        if not created and (
+            record.sha384 != content.custom_metadata["sha384"]
+            or record.len != int(content.custom_metadata["len"])
+            or record.deleted is not None
+        ):
             record.sha384 = content.custom_metadata["sha384"]
             record.len = int(content.custom_metadata["len"])
             record.modified = now
@@ -160,7 +164,7 @@ def _delete_stored_object(self, name) -> Optional[StoredObject]:
         else:
             now = timezone.now()
             # Note that existing_record is a queryset that will have one matching object
-            existing_record.filter(deleted__isnull=True).update(deleted=now)
+            existing_record.exclude_deleted().update(deleted=now)
         return existing_record.first()
 
     def _save(self, name, content):

ietf/doc/storage_utils.py

@@ -12,6 +12,14 @@
 from ietf.utils.log import log
 
 
+class StorageUtilsError(Exception):
+    pass
+
+
+class AlreadyExistsError(StorageUtilsError):
+    pass
+
+
 def _get_storage(kind: str) -> Storage:
     if kind in settings.ARTIFACT_STORAGE_NAMES:
         return storages[kind]
@@ -70,7 +78,7 @@ def store_file(
             # debug.show('f"Asked to store {name} in {kind}: is_new={is_new}, allow_overwrite={allow_overwrite}"')
             if not allow_overwrite and not is_new:
                 debug.show('f"Failed to save {kind}:{name} - name already exists in store"')
-                raise RuntimeError(f"Failed to save {kind}:{name} - name already exists in store")
+                raise AlreadyExistsError(f"Failed to save {kind}:{name} - name already exists in store")
             new_name = _get_storage(kind).save(
                 name,
                 StoredObjectFile(
@@ -85,7 +93,7 @@ def store_file(
             if new_name != name:
                 complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead."
                 debug.show("complaint")
-                raise RuntimeError(complaint)
+                raise StorageUtilsError(complaint)
         except Exception as err:
             log(f"Blobstore Error: Failed to store file {kind}:{name}: {repr(err)}")
             if settings.SERVER_MODE == "development":

ietf/doc/views_material.py

@@ -22,6 +22,7 @@
 from ietf.doc.utils import add_state_change_event, check_common_doc_name_rules
 from ietf.group.models import Group
 from ietf.group.utils import can_manage_materials
+from ietf.meeting.utils import resolve_uploaded_material
 from ietf.utils import log
 from ietf.utils.decorators import ignore_view_kwargs
 from ietf.utils.meetecho import MeetechoAPIError, SlidesManager
@@ -179,6 +180,9 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None):
                             "There was an error creating a hardlink at %s pointing to %s: %s"
                             % (ftp_filepath, filepath, ex)
                         )
+                else:
+                    for meeting in set([s.meeting for s in doc.session_set.all()]):
+                        resolve_uploaded_material(meeting=meeting, doc=doc)
 
             if prev_rev != doc.rev:
                 e = NewRevisionDocEvent(type="new_revision", doc=doc, rev=doc.rev)

ietf/meeting/resources.py

@@ -11,11 +11,15 @@
 
 from ietf import api
 
-from ietf.meeting.models import ( Meeting, ResourceAssociation, Constraint, Room, Schedule, Session,
-                                TimeSlot, SchedTimeSessAssignment, SessionPresentation, FloorPlan,
-                                UrlResource, ImportantDate, SlideSubmission, SchedulingEvent,
-                                BusinessConstraint, ProceedingsMaterial, MeetingHost, Attended,
-                                Registration, RegistrationTicket)
+from ietf.meeting.models import (Meeting, ResourceAssociation, Constraint, Room,
+                                 Schedule, Session,
+                                 TimeSlot, SchedTimeSessAssignment, SessionPresentation,
+                                 FloorPlan,
+                                 UrlResource, ImportantDate, SlideSubmission,
+                                 SchedulingEvent,
+                                 BusinessConstraint, ProceedingsMaterial, MeetingHost,
+                                 Attended,
+                                 Registration, RegistrationTicket)
 
 from ietf.name.resources import MeetingTypeNameResource
 class MeetingResource(ModelResource):

ietf/meeting/tasks.py

@@ -1,13 +1,20 @@
-# Copyright The IETF Trust 2024, All Rights Reserved
+# Copyright The IETF Trust 2024-2025, All Rights Reserved
 #
 # Celery task definitions
 #
+import datetime
+
 from celery import shared_task
+# from django.db.models import QuerySet
 from django.utils import timezone
 
 from ietf.utils import log
 from .models import Meeting
-from .utils import generate_proceedings_content
+from .utils import (
+    generate_proceedings_content,
+    resolve_materials_for_one_meeting,
+    store_blobs_for_one_meeting,
+)
 from .views import generate_agenda_data
 from .utils import fetch_attendance_from_meetings
 
@@ -61,3 +68,123 @@ def fetch_meeting_attendance_task():
                     meeting_stats['processed']
                 )
             )
+
+
+def _select_meetings(
+    meetings: list[str] | None = None,
+    meetings_since: str | None = None,
+    meetings_until: str | None = None
+):  # nyah
+    """Select meetings by number or date range"""
+    # IETF-1 = 1986-01-16
+    EARLIEST_MEETING_DATE = datetime.datetime(1986, 1, 1)
+    meetings_since_dt: datetime.datetime | None = None
+    meetings_until_dt: datetime.datetime | None = None
+
+    if meetings_since == "zero":
+        meetings_since_dt = EARLIEST_MEETING_DATE
+    elif meetings_since is not None:
+        try:
+            meetings_since_dt = datetime.datetime.fromisoformat(meetings_since)
+        except ValueError:
+            log.log(
+                "Failed to parse meetings_since='{meetings_since}' with fromisoformat"
+            )
+            raise
+
+    if meetings_until is not None:
+        try:
+            meetings_until_dt = datetime.datetime.fromisoformat(meetings_until)
+        except ValueError:
+            log.log(
+                "Failed to parse meetings_until='{meetings_until}' with fromisoformat"
+            )
+            raise
+        if meetings_since_dt is None:
+            # if we only got meetings_until, start from the first meeting
+            meetings_since_dt = EARLIEST_MEETING_DATE
+
+    if meetings is None:
+        if meetings_since_dt is None:
+            log.log("No meetings requested, doing nothing.")
+            return Meeting.objects.none()
+        meetings_qs = Meeting.objects.filter(date__gte=meetings_since_dt)
+        if meetings_until_dt is not None:
+            meetings_qs = meetings_qs.filter(date__lte=meetings_until_dt)
+            log.log(
+                "Selecting meetings between "
+                f"{meetings_since_dt} and {meetings_until_dt}"
+            )
+        else:
+            log.log(f"Selecting meetings since {meetings_since_dt}")
+    else:
+        if meetings_since_dt is not None:
+            log.log(
+                "Ignoring meetings_since and meetings_until "
+                "because specific meetings were requested."
+            )
+        meetings_qs = Meeting.objects.filter(number__in=meetings)
+    return meetings_qs
+
+
+@shared_task
+def resolve_meeting_materials_task(
+    *,  # only allow kw arguments
+    meetings: list[str] | None=None,
+    meetings_since: str | None=None,
+    meetings_until: str | None=None
+):
+    """Run materials resolver on meetings
+
+    Can request a set of meetings by number by passing a list in the meetings arg, or
+    by range by passing an iso-format timestamps in meetings_since / meetings_until.
+    To select all meetings, set meetings_since="zero" and omit other parameters. 
+    """
+    meetings_qs = _select_meetings(meetings, meetings_since, meetings_until)
+    for meeting in meetings_qs.order_by("date"):
+        log.log(
+            f"Resolving materials for {meeting.type_id} "
+            f"meeting {meeting.number} ({meeting.date})..."
+        )
+        mark = timezone.now()
+        try:
+            resolve_materials_for_one_meeting(meeting)
+        except Exception as err:
+            log.log(
+                "Exception raised while resolving materials for "
+                f"meeting {meeting.number}: {err}"
+            )
+        else:
+            log.log(f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds.")
+
+
+@shared_task
+def store_meeting_materials_as_blobs_task(
+    *,  # only allow kw arguments
+    meetings: list[str] | None = None,
+    meetings_since: str | None = None,
+    meetings_until: str | None = None
+):
+    """Push meeting materials into the blob store
+
+    Can request a set of meetings by number by passing a list in the meetings arg, or
+    by range by passing an iso-format timestamps in meetings_since / meetings_until.
+    To select all meetings, set meetings_since="zero" and omit other parameters. 
+    """
+    meetings_qs = _select_meetings(meetings, meetings_since, meetings_until)
+    for meeting in meetings_qs.order_by("date"):
+        log.log(
+            f"Creating blobs for materials for {meeting.type_id} "
+            f"meeting {meeting.number} ({meeting.date})..."
+        )
+        mark = timezone.now()
+        try:
+            store_blobs_for_one_meeting(meeting)
+        except Exception as err:
+            log.log(
+                "Exception raised while creating blobs for "
+                f"meeting {meeting.number}: {err}"
+            )
+        else:
+            log.log(
+                f"Blobs created in {(timezone.now() - mark).total_seconds():0.3f} seconds.")