ci: merge main to release (ietf-tools#8152)

Author: rjsparks

ietf/meeting/tasks.py

@@ -0,0 +1,12 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+#
+# Celery task definitions
+#
+from celery import shared_task
+
+from .views import generate_agenda_data
+
+
+@shared_task
+def agenda_data_refresh():
+    generate_agenda_data(force_refresh=True)

ietf/meeting/tests_views.py

@@ -26,6 +26,7 @@
 from django.urls import reverse as urlreverse
 from django.conf import settings
 from django.contrib.auth.models import User
+from django.core.serializers.json import DjangoJSONEncoder
 from django.test import Client, override_settings
 from django.db.models import F, Max
 from django.http import QueryDict, FileResponse
@@ -49,7 +50,7 @@
 from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting
 from ietf.meeting.utils import create_recording, get_next_sequence, bluesheet_data
 from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule
-from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose
+from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose, generate_agenda_data
 from ietf.name.models import SessionStatusName, ImportantDateName, RoleName, ProceedingsMaterialTypeName
 from ietf.utils.decorators import skip_coverage
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
@@ -245,30 +246,34 @@ def test_meeting_agenda(self):
 
         # Agenda API tests
         # -> Meeting data
-        r = self.client.get(urlreverse("ietf.meeting.views.api_get_agenda_data", kwargs=dict(num=meeting.number)))
-        self.assertEqual(r.status_code, 200)  
-        rjson = json.loads(r.content.decode("utf8"))
-        self.assertJSONEqual(
-            r.content.decode("utf8"),
+        # First, check that the generation function does the right thing
+        generated_data = generate_agenda_data(meeting.number)
+        self.assertEqual(
+            generated_data,
             {
                 "meeting": {
                     "number": meeting.number,
                     "city": meeting.city,
                     "startDate": meeting.date.isoformat(),
                     "endDate": meeting.end_date().isoformat(),
-                    "updated": rjson.get("meeting").get("updated"), # Just expect the value to exist
+                    "updated": generated_data.get("meeting").get("updated"),  # Just expect the value to exist
                     "timezone": meeting.time_zone,
                     "infoNote": meeting.agenda_info_note,
                     "warningNote": meeting.agenda_warning_note
                 },
-                "categories": rjson.get("categories"), # Just expect the value to exist
+                "categories": generated_data.get("categories"),  # Just expect the value to exist
                 "isCurrentMeeting": True,
-                "usesNotes": False, # make_meeting_test_data sets number=72
-                "schedule": rjson.get("schedule"), # Just expect the value to exist
+                "usesNotes": False,  # make_meeting_test_data sets number=72
+                "schedule": generated_data.get("schedule"),  # Just expect the value to exist
                 "floors": []
             }
         )
-        # -> Session Materials
+        with patch("ietf.meeting.views.generate_agenda_data", return_value=generated_data):
+            r = self.client.get(urlreverse("ietf.meeting.views.api_get_agenda_data", kwargs=dict(num=meeting.number)))
+        self.assertEqual(r.status_code, 200)  
+        # json.dumps using the DjangoJSONEncoder to handle timestamps consistently
+        self.assertJSONEqual(r.content.decode("utf8"), json.dumps(generated_data, cls=DjangoJSONEncoder))
+        # -> Session MaterialM
         r = self.client.get(urlreverse("ietf.meeting.views.api_get_session_materials", kwargs=dict(session_id=session.id)))
         self.assertEqual(r.status_code, 200)  
         rjson = json.loads(r.content.decode("utf8"))

ietf/meeting/views.py

@@ -25,6 +25,7 @@
 from wsgiref.handlers import format_date_time
 
 from django import forms
+from django.core.cache import caches
 from django.shortcuts import render, redirect, get_object_or_404
 from django.http import (HttpResponse, HttpResponseRedirect, HttpResponseForbidden,
                          HttpResponseNotFound, Http404, HttpResponseBadRequest,
@@ -1657,8 +1658,16 @@ def agenda(request, num=None, name=None, base=None, ext=None, owner=None, utc=""
         }
     })
 
-@cache_page(5 * 60)
-def api_get_agenda_data (request, num=None):
+
+def generate_agenda_data(num=None, force_refresh=False):
+    """Generate data for the api_get_agenda_data endpoint
+    
+    :num: meeting number
+    :force_refresh: True to force a refresh of the cache
+    """
+    cache = caches["default"]
+    cache_timeout = 6 * 60
+
     meeting = get_ietf_meeting(num)
     if meeting is None:
         raise Http404("No such full IETF meeting")
@@ -1667,6 +1676,12 @@ def api_get_agenda_data (request, num=None):
     else:
         pass
 
+    cache_key = f"generate_agenda_data_{meeting.number}"
+    if not force_refresh:
+        cached_value = cache.get(cache_key)
+        if cached_value is not None:
+            return cached_value
+
     # Select the schedule to show
     schedule = get_schedule(meeting, None)
 
@@ -1685,10 +1700,8 @@ def api_get_agenda_data (request, num=None):
 
     # Get Floor Plans
     floors = FloorPlan.objects.filter(meeting=meeting).order_by('order')
-
-    #debug.show('all([(item.acronym,item.session.order_number,item.session.order_in_meeting()) for item in filtered_assignments])')
-
-    return JsonResponse({
+    
+    result = {
         "meeting": {
             "number": schedule.meeting.number,
             "city": schedule.meeting.city,
@@ -1704,7 +1717,13 @@ def api_get_agenda_data (request, num=None):
         "usesNotes": meeting.uses_notes(),
         "schedule": list(map(agenda_extract_schedule, filtered_assignments)),
         "floors": list(map(agenda_extract_floorplan, floors))
-    })
+    }
+    cache.set(cache_key, result, timeout=cache_timeout)
+    return result
+
+
+def api_get_agenda_data(request, num=None):
+    return JsonResponse(generate_agenda_data(num, force_refresh=False))
 
 
 def api_get_session_materials(request, session_id=None):

ietf/stats/views.py

@@ -596,14 +596,17 @@ def build_document_stats_url(stats_type_override=Ellipsis, get_overrides=None):
 
         elif any(stats_type == t[0] for t in possible_yearly_stats_types):
 
-            person_filters = Q(documentauthor__document__type="draft")
-
             # filter persons
             rfc_state = State.objects.get(type="rfc", slug="published")
             if document_type == "rfc":
+                person_filters = Q(documentauthor__document__type="rfc")
                 person_filters &= Q(documentauthor__document__states=rfc_state)
             elif document_type == "draft":
+                person_filters = Q(documentauthor__document__type="draft")
                 person_filters &= ~Q(documentauthor__document__states=rfc_state)
+            else:
+                person_filters = Q(documentauthor__document__type="rfc")
+                person_filters |= Q(documentauthor__document__type="draft")
 
             doc_years = defaultdict(set)
 
@@ -625,8 +628,8 @@ def build_document_stats_url(stats_type_override=Ellipsis, get_overrides=None):
                         ).values_list("source", flat=True)[:1]
                     )
                 )
-                .values_list("draft", "time")
-                .order_by("draft")
+                .values_list("doc", "time")
+                .order_by("doc")
             )
 
             for doc_id, time in rfcevent_qs.iterator():
@@ -663,9 +666,11 @@ def build_document_stats_url(stats_type_override=Ellipsis, get_overrides=None):
                 for name, affiliation, doc in name_affiliation_doc_set:
                     a = aliases.get(affiliation, affiliation)
                     if a:
-                        for year in doc_years.get(doc):
-                            if years_from <= year <= years_to:
-                                bins[(year, a)].add(name)
+                        years = doc_years.get(doc)
+                        if years:
+                            for year in years:
+                                if years_from <= year <= years_to:
+                                    bins[(year, a)].add(name)
 
                 add_labeled_top_series_from_bins(chart_data, bins, limit=8)
 
@@ -724,9 +729,11 @@ def build_document_stats_url(stats_type_override=Ellipsis, get_overrides=None):
                     continent_name = country_to_continent.get(country_name, "")
 
                     if continent_name:
-                        for year in doc_years.get(doc):
-                            if years_from <= year <= years_to:
-                                bins[(year, continent_name)].add(name)
+                        years = doc_years.get(doc)
+                        if years:
+                            for year in years:
+                                if years_from <= year <= years_to:
+                                    bins[(year, continent_name)].add(name)
 
                 add_labeled_top_series_from_bins(chart_data, bins, limit=8)
 

ietf/templates/base.html

@@ -46,7 +46,7 @@
                     {% endif %}
                 </a>
                 <div class="collapse navbar-collapse" id="navbar-collapse">
-                    <ul class="nav navbar-nav">
+                    <ul class="nav navbar-nav flex-nowrap">
                         {% include "base/menu.html" with flavor="top" %}
                     </ul>
                 </div>