From e3e080fa2341d4cae501dcfbebf8e438e49b6f46 Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Tue, 28 Apr 2026 19:02:07 +0200 Subject: [PATCH 1/2] feat: add some per country/affiliation meeting registration statistics (#10550) * Draft for meeting registrations * Add totals, + nicer JS code * Coherent URL parameters * Handle error case when per-continent stats is requested * Remove redundant code * Dynamic get the current IETF meeting * Display %-age when hovering * Add test for meeting statistics * Add test * Add statistics per affiliation * More code coverage for test * Nicer canonical affiliation * Allow navigation by buttons * Also add participants count in the legend * Add timeline over meetings (total and per country) * Default index refers to current meeting stats by number * Remove unused JS code * Add test coverage for timeline statistics * No need to import test coverage * Use stacked lines of onsite/remote when displaying the total timeline * fix a comment * Add timeline for affiliation * Expanding the test coverage to affiliation timeline * Remove unused botocore (unsure how it was added though) * Remove unused package * Code clean-up, add pan & zoom on timelines * Fix button type * refactor: avoid inline JS; safer JSON handling * chore: lint * feat: cache timeline stats Pins the top_n parameter, which had not been plumbed to be dynamically adjustable. * chore: timeout->settings + drop stale setting STATS_NAMES_LIMIT does not appear anywhere else in the codebase * refactor: wait for DOMContentLoaded + restyle * fix: fix null checks * test: update test_meeting_stats() --------- Co-authored-by: Jennifer Richards --- ietf/settings.py | 3 +- ietf/static/js/meeting_stats.js | 57 ++ ietf/static/js/meeting_timeline.js | 84 +++ ietf/stats/tests.py | 63 +- ietf/stats/urls.py | 5 +- ietf/stats/views.py | 612 ++++++++++++++++++-- ietf/templates/base/menu.html | 11 +- ietf/templates/stats/index.html | 8 +- ietf/templates/stats/meeting_stats.html | 61 ++ ietf/templates/stats/meetings_timeline.html | 78 +++ package.json | 2 + 11 files changed, 916 insertions(+), 68 deletions(-) create mode 100644 ietf/static/js/meeting_stats.js create mode 100644 ietf/static/js/meeting_timeline.js create mode 100644 ietf/templates/stats/meeting_stats.html create mode 100644 ietf/templates/stats/meetings_timeline.html diff --git a/ietf/settings.py b/ietf/settings.py index 3aa45a453c..50e069ff1a 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -231,6 +231,7 @@ AGENDA_CACHE_TIMEOUT_DEFAULT = 8 * 24 * 60 * 60 # 8 days AGENDA_CACHE_TIMEOUT_CURRENT_MEETING = 6 * 60 # 6 minutes + WSGI_APPLICATION = "ietf.wsgi.application" AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', ) @@ -1270,7 +1271,7 @@ def skip_unreadable_post(record): except ImportError: pass -STATS_NAMES_LIMIT = 25 +STATS_TIMELINE_CACHE_TIMEOUT = 86400 UTILS_MEETING_CONFERENCE_DOMAINS = ['webex.com', 'zoom.us', 'jitsi.org', 'meetecho.com', 'gather.town', ] UTILS_TEST_RANDOM_STATE_FILE = '.factoryboy_random_state' diff --git a/ietf/static/js/meeting_stats.js b/ietf/static/js/meeting_stats.js new file mode 100644 index 0000000000..70b18a0f03 --- /dev/null +++ b/ietf/static/js/meeting_stats.js @@ -0,0 +1,57 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +document.addEventListener('DOMContentLoaded', () => { + // Need to use autocolors plug-in else all slices are gray... + const autocolors = window['chartjs-plugin-autocolors'] + Chart.register(autocolors) + // ── Safely parse JSON data injected from Django view ── + const totalChartData = JSON.parse(document.getElementById('total-chart-data').textContent) + const inPersonChartData = JSON.parse(document.getElementById('in-person-chart-data').textContent) + + function displayChart (id, data) { + const ctx = document.getElementById(id).getContext('2d') + new Chart(ctx, { + type: 'pie', // Change to 'doughnut' for a donut chart + data: data, + options: { + responsive: true, + plugins: { + autocolors: { + mode: 'data' // Required for Pie charts to color individual slices + }, + legend: { + position: 'bottom', + labels: { + padding: 20, + font: { size: 13 }, + color: '#475569', + generateLabels: function (chart) { + const dataset = chart.data.datasets[0] + return chart.data.labels.map((label, i) => ({ + text: `${label}: ${dataset.data[i]}`, + fillStyle: dataset.backgroundColor[i], + hidden: false, + index: i, + })) + } + } + }, + tooltip: { + callbacks: { + label: function (context) { + const label = context.label || '' + const value = context.raw + const total = context.dataset.data.reduce((a, b) => a + b, 0) + const percentage = ((value / total) * 100).toFixed(1) + + return `${label}: ${value} (${percentage}%)` + } + } + } + } + } + }) + } + + displayChart('totalRegistrationChart', totalChartData) + displayChart('inPersonRegistrationChart', inPersonChartData) +}) diff --git a/ietf/static/js/meeting_timeline.js b/ietf/static/js/meeting_timeline.js new file mode 100644 index 0000000000..161cead0ec --- /dev/null +++ b/ietf/static/js/meeting_timeline.js @@ -0,0 +1,84 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +document.addEventListener('DOMContentLoaded', () => { + // ── Safely parse JSON data injected from Django view ── + const totalChartData = JSON.parse(document.getElementById('total-chart-data').textContent) + const inPersonChartData = JSON.parse(document.getElementById('in-person-chart-data').textContent) + const statsType = JSON.parse(document.getElementById('stats-type-data').textContent) + const stackedLines = statsType === 'total' + + function displayChart (id, data) { + const ctx = document.getElementById(id).getContext('2d') + return new Chart(ctx, { + type: 'line', // Change to 'doughnut' for a donut chart + data: data, + options: { + responsive: true, + scales: { + y: { + stacked: stackedLines, + }, + x: { + title: { + display: true, + text: 'IETF Meeting Number', + }, + }, + }, + plugins: { + legend: { + position: 'bottom', + labels: { + usePointStyle: true, + padding: 15, + font: { size: 12 }, + }, + }, + tooltip: { + backgroundColor: 'rgba(0,0,0,0.8)', + titleFont: { size: 14 }, + bodyFont: { size: 13 }, + callbacks: { + title: function (items) { + return `IETF Meeting ${items[0].label}` + }, + label: function (context) { + return ` ${context.dataset.label}: ${context.parsed.y} participants` + } + } + }, + zoom: { + zoom: { + wheel: { enabled: true }, // scroll to zoom + pinch: { enabled: true }, // pinch on mobile + drag: { enabled: true }, // drag to select range + mode: 'xy', // zoom X-axis and Y-axis + }, + pan: { + enabled: true, + mode: 'xy', // pan X-axis and Y-axis + }, + }, + } + } + }) + } + + const totalChart = displayChart('totalRegistrationChart', totalChartData) + if (inPersonChartData !== null) { + inPersonChart = displayChart('inPersonRegistrationChart', inPersonChartData) + } + document.addEventListener('keydown', (event) => { + if (event.key === 'Escape') { + totalChart.resetZoom() + if (inPersonChart !== null) { + inPersonChart.resetZoom() + } + } + }) + document.getElementById('resetButton').addEventListener('click', () => { + totalChart.resetZoom() + if (inPersonChart !== null) { + inPersonChart.resetZoom() + } + }) +}) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 48552c8fba..373f06e343 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -4,12 +4,14 @@ import calendar import json +import datetime from pyquery import PyQuery import debug # pyflakes:ignore from django.urls import reverse as urlreverse +from django.utils import timezone from ietf.utils.test_utils import login_testing_unauthorized, TestCase import ietf.stats.views @@ -18,24 +20,73 @@ from ietf.group.factories import RoleFactory from ietf.person.factories import PersonFactory from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory +from ietf.meeting.tests_models import MeetingFactory, RegistrationFactory from ietf.utils.timezone import date_today class StatisticsTests(TestCase): def test_stats_index(self): + # Create a meeting as the index page needs to know the current meeting + MeetingFactory(type_id='ietf', number='124', date=timezone.now()) url = urlreverse(ietf.stats.views.stats_index) r = self.client.get(url) self.assertEqual(r.status_code, 200) def test_document_stats(self): - r = self.client.get(urlreverse("ietf.stats.views.document_stats")) - self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index")) - + # Create a meeting as the index page needs to know the current meeting + MeetingFactory(type_id='ietf', number='124', date=timezone.now()) + r = self.client.get(urlreverse(ietf.stats.views.document_stats)) + self.assertRedirects(r, urlreverse(ietf.stats.views.stats_index)) def test_meeting_stats(self): - r = self.client.get(urlreverse("ietf.stats.views.meeting_stats")) - self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index")) - + meeting124 = MeetingFactory(type_id='ietf', number='124', date=timezone.now()) + meeting125 = MeetingFactory(type_id='ietf', number='125', date=timezone.now() + datetime.timedelta(days=120)) + RegistrationFactory.create_batch(15, meeting=meeting124, with_ticket={'attendance_type_id': 'onsite'}, attended=True) + RegistrationFactory(meeting=meeting124, with_ticket={'attendance_type_id': 'onsite'}, attended=False) + RegistrationFactory.create_batch(14, meeting=meeting124, with_ticket={'attendance_type_id': 'remote'}, attended=True) + RegistrationFactory(meeting=meeting124, with_ticket={'attendance_type_id': 'remote'}, attended=False) + RegistrationFactory.create_batch(15, meeting=meeting125, affiliation='Test LLC', with_ticket={'attendance_type_id': 'remote'}, attended=False) + RegistrationFactory.create_batch(25, meeting=meeting125, affiliation='Example, Ltd', with_ticket={'attendance_type_id': 'onsite'}, attended=False) + # Test the meeting specific statitistics per affiliation and per country + r = self.client.get(urlreverse(ietf.stats.views.meeting_stats, kwargs={"meeting_number": "124", "stats_type": "affiliation"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Total Registrations by Affiliation (31 in total)") + self.assertContains(r, "In Person Registrations by Affiliation (16 in total)") + self.assertContains(r, "/stats/meeting/124/affiliation") + self.assertContains(r, "/stats/meeting/125/affiliation") + r = self.client.get(urlreverse(ietf.stats.views.meeting_stats, kwargs={"meeting_number": "124", "stats_type": "country"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Total Registrations by Country (31 in total)") + self.assertContains(r, "In Person Registrations by Country (16 in total)") + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + # Test the meetings timeline per country + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "country"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + self.assertContains(r, "This page provides a timeline of meeting registrations by country") + # Test the meetings timeline per affiliation + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "affiliation"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/affiliation") + self.assertContains(r, "/stats/meeting/125/affiliation") + self.assertContains(r, "This page provides a timeline of meeting registrations by affiliation") + # Extract the JSON embedded in the response + pq = PyQuery(r.content) + in_person_data = json.loads(pq.find("script#in-person-chart-data").text()) + self.assertTrue( + any( + ds["label"] == "Example" and ds["data"] == [0, 25] + for ds in in_person_data["datasets"] + ) + ) + # Test the global meetings timeline + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "total"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + self.assertContains(r, "This page provides a timeline of meeting registrations.") def test_known_country_list(self): # check redirect diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index d2993759d2..01b8758c84 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -11,7 +11,8 @@ url(r"^$", views.stats_index), url(r"^document/(?:(?Pauthors|pages|words|format|formlang|author/(?:documents|affiliation|country|continent|citations|hindex)|yearly/(?:affiliation|country|continent))/)?$", views.document_stats), url(r"^knowncountries/$", views.known_countries_list), - url(r"^meeting/(?P\d+)/(?Pcountry|continent)/$", views.meeting_stats), - url(r"^meeting/(?:(?Poverview|country|continent)/)?$", views.meeting_stats), + url(r"^meeting/$", views.meetings_timeline), + url(r"^meeting/(?P\d+)/(?Paffiliation|country)/$", views.meeting_stats), + url(r"^meeting/(?:(?Paffiliation|country|total)/)?$", views.meetings_timeline), url(r"^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), ] diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 504d84e86d..d61b673075 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -9,11 +9,13 @@ import dateutil.relativedelta from collections import defaultdict +from django.conf import settings from django.contrib.auth.decorators import login_required +from django.core.cache import cache from django.http import HttpResponseRedirect from django.shortcuts import render from django.urls import reverse as urlreverse - +from django.db.models import Count import debug # pyflakes:ignore @@ -25,15 +27,32 @@ from ietf.group.models import Role, Group from ietf.person.models import Person from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName +from ietf.meeting.models import Registration from ietf.ietfauth.utils import has_role from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, DEADLINE_TZINFO +from ietf.meeting.helpers import get_current_ietf_meeting_num, get_ietf_meeting +# Color palette for lines +colors = [ + '#FF6384', '#36A2EB', '#FFCE56', '#4BC0C0', '#9966FF', + '#FF9F40', '#C9CBCF', '#7BC043', '#F37735', '#00ABA9', + '#2B5797', '#E81123', '#00A4EF', '#7FBA00', '#FFB900', + '#D83B01', '#B4009E', '#5C2D91', '#008575', '#E3008C', +] def stats_index(request): - return render(request, "stats/index.html") + """Render the statistics index page with the current meeting number as it is required by the meeting menu item.""" + current_meeting = get_current_ietf_meeting_num() + return render(request, "stats/index.html", { + "current_meeting": current_meeting + }) def generate_query_string(query_dict, overrides): + """ + Returns: + A query string starting with '?' if there are parameters, empty string otherwise. + """ query_part = "" if query_dict or overrides: @@ -58,9 +77,20 @@ def generate_query_string(query_dict, overrides): return query_part def get_choice(request, get_parameter, possible_choices, multiple=False): - # the statistics are built with links to make navigation faster, - # so we don't really have a form in most cases, so just use this - # helper instead to select between the choices + """Extract a choice from the request GET parameters. + + Since statistics pages use links for navigation instead of forms, + this helper selects between possible choices from the URL parameters. + + Args: + request: The HTTP request object. + get_parameter: The name of the GET parameter. + possible_choices: List of tuples (value, label). + multiple: If True, return a list of found values; otherwise return the first found or None. + + Returns: + The selected value(s) or None. + """ values = request.GET.getlist(get_parameter) found = [t[0] for t in possible_choices if t[0] in values] @@ -73,75 +103,553 @@ def get_choice(request, get_parameter, possible_choices, multiple=False): return None def add_url_to_choices(choices, url_builder): + """Add URLs to a list of choices. + + Args: + choices: List of tuples (slug, label). + url_builder: Function that takes a slug and returns a URL. + + Returns: + List of tuples (slug, label, url). + """ return [ (slug, label, url_builder(slug)) for slug, label in choices] -def put_into_bin(value, bin_size): - if value is None: - return (0, '') +def document_stats(request, stats_type=None): + # timeline per year, or per specific year: streams, affiliation, rfc vs I-D + # could also be time between individual/WG I-D to rfc publication/IESG ballot + # DISCUSS resolution time + # Humm also split by authors (affiliation) / documents (the rest) probably + """Redirect to the stats index page. Deprecated view.""" + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) - v = (value // bin_size) * bin_size - return (v, "{} - {}".format(v, v + bin_size - 1)) +def known_countries_list(request, stats_type=None, acronym=None): + """Render a list of known countries with their aliases.""" + countries = CountryName.objects.prefetch_related("countryalias_set") + for c in countries: + # the sorting is a bit of a hack - it puts the ISO code first + # since it was added in a migration + c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) -def prune_unknown_bin_with_known(bins): - # remove from the unknown bin all authors within the - # named/known bins - all_known = { n for b, names in bins.items() if b for n in names } - bins[""] = [name for name in bins[""] if name not in all_known] - if not bins[""]: - del bins[""] + return render(request, "stats/known_countries_list.html", { + "countries": countries, + }) -def count_bins(bins): - return len({ n for b, names in bins.items() if b for n in names }) +def canonicalize_affiliation(affiliation): + """Canonicalize an affiliation string by removing common suffixes and standardizing prefixes. + + Args: + affiliation: The affiliation string to canonicalize. + + Returns: + The canonicalized affiliation string, or None if input is None. + """ + if not affiliation or affiliation.lower() in ('n/a', 'none', 'unspecified'): + return None + for suffix in ('ab', 'ag', 'corp', 'corp.', 'corporation', 'gmbh', 'inc.', 'inc', 'international pte ltd', 'llc', 'ltd', 'ltd.', 'private limited', 'pty ltd', 'pvt ltd'): + if affiliation.lower().endswith(', ' + suffix): + affiliation = affiliation[:-(len(suffix)+2)] + elif affiliation.lower().endswith(' ' + suffix): + affiliation = affiliation[:-(len(suffix)+1)] + elif affiliation.lower().endswith(',' + suffix): + affiliation = affiliation[:-(len(suffix)+1)] + for prefix in ('akamai','apple', 'cisco', 'futurewei', 'google', 'hitachi', 'hpe', 'huawei', 'juniper', 'meta', 'nokia', 'ntt', 'siemens'): + if affiliation.lower().startswith(prefix + ' '): + affiliation = prefix + return affiliation.title() + +def get_affiliation_data_for_meetings(attendance_type=None): + """Get affiliation participation data for meetings timeline chart. + + Args: + attendance_type: Optional filter for attendance type (e.g., 'onsite'). + + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = f'stats:get_affiliation_data_for_meetings:{attendance_type}' + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + top_n = 20 # could be a parameter, but would need to adjust cache handling + + # Get registration status details + if attendance_type: + registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) + else: + registrations = Registration.objects.all() + registrations = registrations.values('affiliation', 'meeting__number') + + # Count per canonicalized affiliation + organization = dict() + meetings_set = set() + org_totals = defaultdict(int) + data_map = defaultdict(dict) # {org: {meeting: count}} + + for reg in registrations: + meeting = reg['meeting__number'] + meetings_set.add(meeting) + affiliation = canonicalize_affiliation(reg['affiliation']) or "Unspecified" + organization[affiliation] = organization.get(affiliation, 0) + 1 + org_totals[affiliation] = org_totals.get(affiliation, 0) + 1 + data_map[affiliation][meeting] = data_map[affiliation].get(meeting, 0) + 1 + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + + # ── Step 3: Get top N countries ── + top_orgs = sorted( + org_totals.keys(), + key=lambda c: org_totals[c], + reverse=True + )[:top_n] + non_top_orgs = org_totals.keys() - top_orgs + other_totals = defaultdict(int) + for m in sorted_meetings: + other_totals[m] = 0 + for c in non_top_orgs: + other_totals[m] += int(data_map[c].get(m, 0)) + + # ── Step 4: Build Chart.js datasets ── + + datasets = [] + for idx, org in enumerate(top_orgs): + color = colors[idx % len(colors)] + datasets.append({ + 'label': org, + 'data': [data_map[org].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'fill': False, + 'tension': 0.3, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + + # -- Step 4.bis handle the other -- + datasets.append({ + 'label': 'Other', + 'data': [other_totals.get(m, 0) for m in sorted_meetings], + 'borderColor': 'black', + 'fill': False, + 'tension': 0.3, + 'pointColor': 'black', + 'pointBackgroundColor': 'black', + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) -def add_labeled_top_series_from_bins(chart_data, bins, limit): - """Take bins on the form (x, label): [name1, name2, ...], figure out - how many there are per label, take the overall top ones and put - them into sorted series like [(x1, len(names1)), (x2, len(names2)), ...].""" - aggregated_bins = defaultdict(set) - xs = set() - for (x, label), names in bins.items(): - xs.add(x) - aggregated_bins[label].update(names) + return sorted_meetings, datasets - xs = list(sorted(xs)) +def get_country_data_for_meetings(attendance_type=None): + """Get country participation data for meetings timeline chart. - sorted_bins = sorted(aggregated_bins.items(), key=lambda t: len(t[1]), reverse=True) - top = [ label for label, names in list(sorted_bins)[:limit]] + Args: + attendance_type: Optional filter for attendance type (e.g., 'onsite'). - for label in top: - series_data = [] + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = f'stats:get_country_data_for_meetings:{attendance_type}' + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + top_n = 10 # could be a parameter, but would need to adjust cache handling + # Get registration status counts, aggregated by country_code + if attendance_type: + registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) + else: + registrations = Registration.objects.all() + queryset = ( + registrations + .values( + 'meeting__number', # e.g. "118", "119", "120" + 'country_code' # country code of the participant + ) + .annotate(participant_count=Count('id')) + .order_by('meeting__number') # chronological order + ) + + # ── Step 1: Collect all meetings and country totals ── + meetings_set = set() + country_totals = defaultdict(int) + data_map = defaultdict(dict) # {country: {meeting: count}} + + for row in queryset: + meeting = row['meeting__number'] + country = row['country_code'] + count = row['participant_count'] + + meetings_set.add(meeting) + country_totals[country] += count + data_map[country][meeting] = count + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + + # ── Step 3: Get top N countries ── + top_countries = sorted( + country_totals.keys(), + key=lambda c: country_totals[c], + reverse=True + )[:top_n] + + # -- Step 3.bis do the 'other' category -- + non_top_countries = country_totals.keys() - top_countries + other_totals = defaultdict(int) + for m in sorted_meetings: + other_totals[m] = 0 + for c in non_top_countries: + other_totals[m] += int(data_map[c].get(m, 0)) + + # ── Step 4: Build Chart.js datasets ── + + datasets = [] + for idx, country in enumerate(top_countries): + color = colors[idx % len(colors)] + datasets.append({ + 'label': country, + 'data': [data_map[country].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'fill': False, + 'tension': 0.3, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + + # -- Step 4.bis handle the other -- + datasets.append({ + 'label': 'Other', + 'data': [other_totals.get(m, 0) for m in sorted_meetings], + 'borderColor': 'black', + 'fill': False, + 'tension': 0.3, + 'pointColor': 'black', + 'pointBackgroundColor': 'black', + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) - for x in xs: - names = bins.get((x, label), set()) + return sorted_meetings, datasets + +def get_data_for_meetings(): + """Get total participation data by attendance type for meetings timeline chart. + + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = "stats:get_data_for_meetings" + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + # Get registration status counts, aggregated by ticket types + registrations = Registration.objects.filter(tickets__attendance_type__in=['onsite', 'remote']) + queryset = ( + registrations + .values( + 'meeting__number', # e.g. "118", "119", "120" + 'tickets__attendance_type' + ) + .annotate(participant_count=Count('id')) + .order_by('meeting__number') # chronological order + ) + + # ── Step 1: Collect all meetings and tickets totals ── + meetings_set = set() + tickets_totals = defaultdict(int) + data_map = defaultdict(dict) # {ticket: {meeting: count}} + + for row in queryset: + meeting = row['meeting__number'] + ticket = row['tickets__attendance_type'] + count = row['participant_count'] + + meetings_set.add(meeting) + tickets_totals[ticket] += count + data_map[ticket][meeting] = count + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + ticket_types = tickets_totals.keys() + + # ── Step 4: Build Chart.js datasets ── + # Color palette for lines + colors = [ '#FF6384', '#36A2EB'] + + datasets = [] + for idx, ticket_type in enumerate(ticket_types): + color = colors[idx % len(colors)] + datasets.append({ + 'label': ticket_type, + 'data': [data_map[ticket_type].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'backgroundColor': color + '99', # 60% opacity fill + 'fill': True, + 'tension': 0.0, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) + return sorted_meetings, datasets + +def meetings_timeline(request, stats_type='country'): + """Render the meetings timeline page with participation statistics over time. + + Args: + request: The HTTP request object. + stats_type: Type of statistics ('country' or 'total'). + top_n: Number of top items to show (for country stats). + + Returns: + Rendered response for the meetings timeline template. + """ + if stats_type == 'total': + total_labels, total_data_sets = get_data_for_meetings() + in_person_labels = ([], []) + in_person_data_sets = ([], []) + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + elif stats_type == 'affiliation': + total_labels, total_data_sets = get_affiliation_data_for_meetings() + in_person_labels, in_person_data_sets = get_affiliation_data_for_meetings(attendance_type='onsite') + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + elif stats_type == 'country': + total_labels, total_data_sets = get_country_data_for_meetings() + in_person_labels, in_person_data_sets = get_country_data_for_meetings(attendance_type='onsite') + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) - series_data.append((x, len(names))) + total_chart_data = { + 'labels': total_labels, + 'datasets': total_data_sets, + } - chart_data.append({ - "data": series_data, - "name": label - }) + # On per country/affiliation have a separate graph for inperson + if stats_type == 'total': + in_person_chart_data = None + else: + in_person_chart_data = { + 'labels': in_person_labels, + 'datasets': in_person_data_sets, + } -def document_stats(request, stats_type=None): - return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + # Prepare the list of choice buttons for the template + possible_stats_types = [ + ("affiliation", "Per affiliation", urlreverse(meetings_timeline, kwargs={'stats_type': 'affiliation'})), + ("country", "Per country", urlreverse(meetings_timeline, kwargs={'stats_type': 'country'})), + ("total", "Total", urlreverse(meetings_timeline, kwargs={'stats_type': 'total'})), + ] + current_meeting = get_current_ietf_meeting_num() + if stats_type == 'total': + possible_stats_type = 'country' + else: + possible_stats_type = stats_type -def known_countries_list(request, stats_type=None, acronym=None): - countries = CountryName.objects.prefetch_related("countryalias_set") - for c in countries: - # the sorting is a bit of a hack - it puts the ISO code first - # since it was added in a migration - c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) + possible_meeting_numbers = [(int(current_meeting)-1, urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting)-1, 'stats_type': possible_stats_type})), + (int(current_meeting), urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting), 'stats_type': possible_stats_type})), + (int(current_meeting)+1, urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting)+1, 'stats_type': possible_stats_type}))] - return render(request, "stats/known_countries_list.html", { - "countries": countries, + return render(request, "stats/meetings_timeline.html", { + "top_n": top_n, + "possible_stats_types": possible_stats_types, + "possible_meeting_numbers": possible_meeting_numbers, + "stats_type": stats_type, + "total_chart_data": total_chart_data, + "in_person_chart_data": in_person_chart_data, }) -def meeting_stats(request, num=None, stats_type=None): - return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) +def get_affiliation_data_for_meeting(meeting_number, minimum_required, attendance_type=None): + """Get affiliation participation data for a specific meeting. + + Args: + meeting_number: The meeting number. + minimum_required: Minimum count to include in main data (others go to 'Other'). + attendance_type: Optional filter for attendance type. + + Returns: + Tuple of (labels, data, total) for chart display. + """ + # Get registration status details + registrations = Registration.objects.filter(meeting__number=meeting_number) + if attendance_type: + registrations = registrations.filter(tickets__attendance_type=attendance_type) + registrations = registrations.values('affiliation') + + # Count per canonicalized affiliation + organization = dict() + for reg in registrations: + affiliation = canonicalize_affiliation(reg['affiliation']) or "Unspecified" + organization[affiliation] = organization.get(affiliation, 0) + 1 + + # Sort to have the largest count first (nicer in pie chart) + sorted_orgs = sorted(organization.items(), key=lambda t: t[1], reverse=True) + labels = [] + data = [] + others_count = 0 + total = 0 + for org, count in sorted_orgs: + total += count + if count > minimum_required: + labels.append(org) + data.append(count) + else: + others_count += count + + if others_count > 0: + labels.append('Other') + data.append(others_count) + + return labels, data, total + +def get_data_for_meeting(meeting_number, minimum_required, attendance_type=None): + """Get country participation data for a specific meeting. + + Args: + meeting_number: The meeting number. + minimum_required: Minimum count to include in main data (others go to 'Other'). + attendance_type: Optional filter for attendance type. + + Returns: + Tuple of (labels, data, total) for chart display. + """ + # Get registration status counts, aggregated by country_code + registration_counts = Registration.objects.filter(meeting__number=meeting_number) + if attendance_type: + registration_counts = registration_counts.filter(tickets__attendance_type=attendance_type) + registration_counts = registration_counts.values('country_code').annotate(count=Count('country_code')).order_by('-count') + + labels = [] + data = [] + others_count = 0 + total = 0 + for item in registration_counts: + total += item['count'] + if item['count'] > minimum_required: + labels.append(item['country_code']) + data.append(item['count']) + else: + others_count += item['count'] + + if others_count > 0: + labels.append('Other') + data.append(others_count) + + return labels, data, total + +def meeting_stats(request, meeting_number=None, stats_type='country'): + """Render statistics for a specific meeting. + + Args: + request: The HTTP request object. + meeting_number: The meeting number (defaults to current). + stats_type: Type of statistics ('country' or 'affiliation'). + + Returns: + Rendered response for the meeting stats template. + """ + + current_meeting = get_current_ietf_meeting_num() + if meeting_number is None: + meeting_number = current_meeting + + this_meeting = get_ietf_meeting(meeting_number) + + if stats_type == 'affiliation': + minimum_required = 4 + total_labels, total_data, total_total = get_affiliation_data_for_meeting(meeting_number, minimum_required) + in_person_labels, in_person_data, in_person_total = get_affiliation_data_for_meeting(meeting_number, minimum_required, attendance_type='onsite') + elif stats_type == 'country': + minimum_required = 10 + total_labels, total_data, total_total = get_data_for_meeting(meeting_number, minimum_required) + in_person_labels, in_person_data, in_person_total = get_data_for_meeting(meeting_number, minimum_required, attendance_type='onsite') + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + + total_chart_data = { + 'labels': total_labels, + 'datasets': [{ + 'label': 'Total Registrations by ' + stats_type, + 'data': total_data, + 'borderColor': '#ffffff', + 'borderWidth': 2, + }] + } + in_person_chart_data = { + 'labels': in_person_labels, + 'datasets': [{ + 'label': 'In Person Registrations by ' + stats_type, + 'data': in_person_data, + 'borderColor': '#ffffff', + 'borderWidth': 2, + }] + } + + # Prepare the list of choice buttons for the template + possible_stats_types = [ + ("affiliation", "Per affiliation", urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': 'affiliation'})), + ("country", "Per country", urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': 'country'})), + ] + + # Prepare the list of meeting number buttons for the template + possible_meeting_numbers = [('All', urlreverse(meetings_timeline, kwargs={'stats_type': stats_type}))] + if int(meeting_number) > 72: # No registration data before IETF-72 + possible_meeting_numbers.append((int(meeting_number)-1, urlreverse(meeting_stats, kwargs={'meeting_number': int(meeting_number)-1, 'stats_type': stats_type}))) + possible_meeting_numbers.append((meeting_number, urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': stats_type}))) + if int(meeting_number) <= int(current_meeting): # Allow current meeting +1 + possible_meeting_numbers.append((int(meeting_number)+1, urlreverse(meeting_stats, kwargs={'meeting_number': int(meeting_number)+1, 'stats_type': stats_type}))) + + return render(request, "stats/meeting_stats.html", { + "meeting_number": meeting_number, + "meeting_date": this_meeting.date, + "meeting_country": this_meeting.country, + "meeting_city": this_meeting.city, + "possible_stats_types": possible_stats_types, + "possible_meeting_numbers": possible_meeting_numbers, + "stats_type": stats_type, + "minimum_required": minimum_required, + "total_chart_data": total_chart_data, + "total_total": total_total, + "in_person_chart_data": in_person_chart_data, + "in_person_total": in_person_total + }) @login_required def review_stats(request, stats_type=None, acronym=None): + """Render review statistics page with tables and charts for review assignments. + + Shows completion status, results, assignment states, and time series data. + Supports both team-level and reviewer-level views with filtering options. + + Args: + request: The HTTP request object. + stats_type: Type of statistics ('completion', 'results', 'states', 'time'). + acronym: Team acronym for reviewer-level view (None for team view). + + Returns: + Rendered response for the review stats template. + """ # This view is a bit complex because we want to show a bunch of # tables with various filtering options, and both a team overview # and a reviewers-within-team overview - and a time series chart. diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html index 8ff6e952da..43ca025e28 100644 --- a/ietf/templates/base/menu.html +++ b/ietf/templates/base/menu.html @@ -428,12 +428,11 @@ Downref registry -
  • - +
  • + Statistics -
      +

      - Statistics on meetings and authorship are not currently available. + Statistics on authorship are not currently available.

      {% endblock %} \ No newline at end of file diff --git a/ietf/templates/stats/meeting_stats.html b/ietf/templates/stats/meeting_stats.html new file mode 100644 index 0000000000..b32f5a4046 --- /dev/null +++ b/ietf/templates/stats/meeting_stats.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} +{% load origin %} +{% origin %} +{% load ietf_filters static django_bootstrap5 %} +{% block js %} + + + + {{ total_chart_data|json_script:"total-chart-data" }} + {{ in_person_chart_data|json_script:"in-person-chart-data" }} + +{% endblock %} +{% block content %} + {% origin %} +

      + {% block title %} + Statistics for IETF-{{ meeting_number }} ({{ meeting_date }}, {{ meeting_city }}, {{ meeting_country }}) Registrations + {% endblock %} +

      +
      + +
      + {% for slug, label, url in possible_stats_types %} + {{ label }} + {% endfor %} +
      + +
      + {% for num, url in possible_meeting_numbers %} + {{ num }} + {% endfor %} +
      +
      +

      + This page provides a visual representation of the total registrations for IETF-{{ meeting_number }} by {{ stats_type }}. + Only categories having more than {{ minimum_required }} registrations are displayed separately, + else they are grouped under "Other". +

      +
      +
      +

      Total Registrations by {{ stats_type|title }} ({{ total_total}} in total)

      +
      + +
      +
      +
      +

      In Person Registrations by {{ stats_type|title }} ({{ in_person_total}} in total)

      +
      + +
      +
      +
      +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/stats/meetings_timeline.html b/ietf/templates/stats/meetings_timeline.html new file mode 100644 index 0000000000..65fb4e09c0 --- /dev/null +++ b/ietf/templates/stats/meetings_timeline.html @@ -0,0 +1,78 @@ +{% extends "base.html" %} +{% load origin %} +{% origin %} +{% load ietf_filters static django_bootstrap5 %} +{% block js %} + + + + + {{ total_chart_data|json_script:"total-chart-data" }} + {{ in_person_chart_data|json_script:"in-person-chart-data" }} + {{ stats_type|json_script:"stats-type-data" }} + +{% endblock %} +{% block content %} + {% origin %} +

      + {% block title %} + Statistics for IETF Meeting Registrations + {% endblock %} +

      + +
      + +
      + {% for slug, label, url in possible_stats_types %} + {{ label }} + {% endfor %} +
      + +
      + {% for num, url in possible_meeting_numbers %} + {{ num }} + {% endfor %} +
      +
      +

      + {% if stats_type == 'total' %} + This page provides a timeline of meeting registrations. + {% else %} + This page provides a timeline of meeting registrations by {{ stats_type }} with a limit of {{ top_n }} categories. + {% endif %} + Panning can be done via the mouse or with a finger. Zooming is done via the mouse wheel or via a pinch gesture. Press ESC + or click to reset panning/zooming. +

      +
      +
      + {% if stats_type == 'total' %} +

      Total Registrations

      + {% else %} +

      Total Registrations by {{ stats_type|title }}

      + {% endif %} +
      + +
      +
      + {% if stats_type != 'total' %} +
      + {% if stats_type == 'total' %} +

      Total In Person Registrations

      + {% else %} +

      In Person Registrations by {{ stats_type|title }}

      + {% endif %} +
      + +
      +
      + {% endif %} +
      +{% endblock %} \ No newline at end of file diff --git a/package.json b/package.json index 57642d7860..6f61aaba29 100644 --- a/package.json +++ b/package.json @@ -146,6 +146,8 @@ "ietf/static/js/manage-community-list.js", "ietf/static/js/manage-review-requests.js", "ietf/static/js/meeting-interim-request.js", + "ietf/static/js/meeting_stats.js", + "ietf/static/js/meeting_timeline.js", "ietf/static/js/moment.js", "ietf/static/js/navbar-doc-search.js", "ietf/static/js/password_strength.js", From 4964901bb5a29d37ab7a8f7ee925eda7cf734b6f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Apr 2026 18:53:12 -0300 Subject: [PATCH 2/2] fix: bind gunicorn on ipv6 interface + config refactor (#10769) * refactor: move gunicorn cfg to python * fix: bind gunicorn on ipv6 interface, too --- dev/build/datatracker-start.sh | 12 +----------- dev/build/gunicorn.conf.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index a676415a26..012a563412 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -45,16 +45,6 @@ cleanup () { trap 'trap "" TERM; cleanup' TERM # start gunicorn in the background so we can trap the TERM signal -gunicorn \ - -c /workspace/gunicorn.conf.py \ - --workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \ - --max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \ - --timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \ - --bind :8000 \ - --log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \ - --capture-output \ - --access-logfile -\ - ${DATATRACKER_GUNICORN_EXTRA_ARGS} \ - ietf.wsgi:application & +gunicorn -c /workspace/gunicorn.conf.py ${DATATRACKER_GUNICORN_EXTRA_ARGS} ietf.wsgi:application & gunicorn_pid=$! wait "${gunicorn_pid}" diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py index 9af4478685..37991a55dc 100644 --- a/dev/build/gunicorn.conf.py +++ b/dev/build/gunicorn.conf.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2024-2025, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved import os import ietf @@ -12,6 +12,22 @@ from opentelemetry.instrumentation.pymemcache import PymemcacheInstrumentor from opentelemetry.instrumentation.requests import RequestsInstrumentor +# Bind interfaces for both ipv4 and ipv6 +bind = ["127.0.0.1:8000", "[::1]:8000"] + +# Disable control socket +control_socket_disable = True + +# Settings configurable via environment +workers = int(os.environ.get("DATATRACKER_GUNICORN_WORKERS", "9")) +max_requests = int(os.environ.get("DATATRACKER_GUNICORN_MAX_REQUESTS", "32768")) +timeout = int(os.environ.get("DATATRACKER_GUNICORN_TIMEOUT", "180")) +loglevel = os.environ.get("DATATRACKER_GUNICORN_LOG_LEVEL", "info") + +# Logging / stdout capture +capture_output = True +accesslog = "-" + # Configure security scheme headers for forwarded requests. Cloudflare sets X-Forwarded-Proto # for us. Don't trust any of the other similar headers. Only trust the header if it's coming # from localhost, as all legitimate traffic will reach gunicorn via co-located nginx.