diff --git a/ietf/secr/telechat/tests.py b/ietf/secr/telechat/tests.py index 91ccde21879..b173c1f785d 100644 --- a/ietf/secr/telechat/tests.py +++ b/ietf/secr/telechat/tests.py @@ -159,6 +159,7 @@ def test_doc_detail_charter(self): def test_bash(self): today = date_today() + TelechatDate.objects.filter(date=today).delete() # Clean up first TelechatDate.objects.create(date=today) url = reverse('ietf.secr.telechat.views.bash',kwargs={'date':today.strftime('%Y-%m-%d')}) self.client.login(username="secretary", password="secretary+password") diff --git a/ietf/static/js/stats_document_timeline.js b/ietf/static/js/stats_document_timeline.js new file mode 100644 index 00000000000..29c7aeccc75 --- /dev/null +++ b/ietf/static/js/stats_document_timeline.js @@ -0,0 +1,89 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +import Chart from 'chart.js/auto' +import zoomPlugin from 'chartjs-plugin-zoom' + +document.addEventListener('DOMContentLoaded', () => { + Chart.register(zoomPlugin) // enable the zoom plugin + + // ── Safely parse JSON data injected from Django view ── + const chartData = JSON.parse(document.getElementById('chart_data').textContent) ; + const objects = JSON.parse(document.getElementById('objects').textContent) ; + const stackedLines = false ; + + function displayChart (id, data) { + const ctx = document.getElementById(id).getContext('2d') ; + return new Chart(ctx, { + type: 'line', + data: data, + options: { + responsive: true, + scales: { + y: { + stacked: stackedLines, + }, + x: { + title: { + display: true, + text: 'Year', + }, + }, + }, + plugins: { + legend: { + position: 'bottom', + labels: { + usePointStyle: true, + padding: 15, + font: { size: 12 }, + }, + }, + tooltip: { + backgroundColor: 'rgba(0,0,0,0.8)', + titleFont: { size: 14 }, + bodyFont: { size: 13 }, + callbacks: { + title: function(items) { + return `${items[0].label}`; + }, + label: function(context) { + return ` ${context.dataset.label}: ${context.parsed.y} ${objects}`; + } + } + }, + zoom: { + zoom: { + wheel: { + enabled: true, + modifierKey: 'alt' // Alt + scroll wheel to zoom + }, // scroll to zoom + pinch: { + enabled: true + }, // pinch on mobile + drag: { // drag to select range + enabled: true, + modifierKey: 'alt' + }, + mode: 'xy', // zoom X-axis and Y-axis + }, + pan: { + enabled: true, + modifierKey: 'alt', + mode: 'xy', // pan X-axis and Y-axis + }, + }, + } + } + }) + } + + const documentsChart = displayChart('documentsChart', chartData) ; + + document.addEventListener('keydown', (event) => { + if (event.key === 'Escape') { + documentsChart.resetZoom() + } + }) + document.getElementById('resetButton').addEventListener('click', () => { + documentsChart.resetZoom() + }) +}) diff --git a/ietf/static/js/stats_document_total.js b/ietf/static/js/stats_document_total.js new file mode 100644 index 00000000000..7e8f0832595 --- /dev/null +++ b/ietf/static/js/stats_document_total.js @@ -0,0 +1,117 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +import Chart from 'chart.js/auto' +import zoomPlugin from 'chartjs-plugin-zoom' + +document.addEventListener('DOMContentLoaded', () => { + Chart.register(zoomPlugin) // enable the zoom plugin + const hidden = new Set(); // track suppressed categories + + // ── Safely parse JSON data injected from Django view ── + const chartData = JSON.parse(document.getElementById('chart_data').textContent) ; + const objects = JSON.parse(document.getElementById('objects').textContent) ; + + function refreshChart(chart) { + // On first call, snapshot the original data onto the chart instance itself + if (!chart._originalData) { + chart._originalData = { + labels: [...chart.data.labels], + values: [...chart.data.datasets[0].data], + colors: Array.isArray(chart.data.datasets[0].backgroundColor) + ? [...chart.data.datasets[0].backgroundColor] + : chart.data.labels.map(() => chart.data.datasets[0].backgroundColor), + }; + } + + const original = chart._originalData; + const labels = [], values = [], colors = []; + + original.labels.forEach((lbl, i) => { + if (!hidden.has(lbl)) { + labels.push(lbl); + values.push(original.values[i]); + colors.push(original.colors[i]); + } + }); + + chart.data.labels = labels; + chart.data.datasets[0].data = values; + chart.data.datasets[0].backgroundColor = colors; + chart.update(); + } + + function displayChart (id, data) { + const ctx = document.getElementById(id).getContext('2d') ; + let chart = new Chart(ctx, { + type: 'bar', + data: data, + options: { + indexAxis: 'y', + onClick: (event, elements) => { + if (elements.length > 0) { + const idx = elements[0].index; + const label = chart.data.labels[idx]; + hidden.add(label); + refreshChart(chart); + } + }, + responsive: true, + scales: { + x: { + title: { + display: true, + text: 'Number of ' + objects, + }, + }, + y: { + ticks: { + autoSkip: false, // Display all labels even if messy... + } + }, + }, + plugins: { + legend: { + display: false, + }, + tooltip: { + backgroundColor: 'rgba(0,0,0,0.8)', + titleFont: { size: 14 }, + bodyFont: { size: 13 }, + callbacks: { + title: function(items) { + return `${items[0].label}`; + }, + label: function(context) { + return `${context.formattedValue} ${objects}`; + } + } + }, + zoom: { + zoom: { + wheel: { + enabled: true, + modifierKey: 'alt' // Alt + scroll wheel to zoom + }, // scroll to zoom + pinch: { + enabled: true + + }, // pinch on mobile + drag: { // drag to select range + enabled: true, + modifierKey: 'alt' + }, + mode: 'xy', // zoom X-axis and Y-axis + }, + pan: { + enabled: true, + modifierKey: 'alt', + mode: 'xy', // pan X-axis and Y-axis + }, + }, + } + } + }) ; + return chart; + } + + displayChart('documentsChart', chartData) ; +}) diff --git a/ietf/static/js/meeting_stats.js b/ietf/static/js/stats_meeting.js similarity index 90% rename from ietf/static/js/meeting_stats.js rename to ietf/static/js/stats_meeting.js index cf43d08eb8a..5944f198e28 100644 --- a/ietf/static/js/meeting_stats.js +++ b/ietf/static/js/stats_meeting.js @@ -1,9 +1,7 @@ // Copyright The IETF Trust 2026, All Rights Reserved import Chart from 'chart.js/auto' -import autocolors from 'chartjs-plugin-autocolors' document.addEventListener('DOMContentLoaded', () => { - Chart.register(autocolors) // ── Safely parse JSON data injected from Django view ── const totalChartData = JSON.parse(document.getElementById('total-chart-data').textContent) const inPersonChartData = JSON.parse(document.getElementById('in-person-chart-data').textContent) @@ -16,9 +14,6 @@ document.addEventListener('DOMContentLoaded', () => { options: { responsive: true, plugins: { - autocolors: { - mode: 'data' // Required for Pie charts to color individual slices - }, legend: { position: 'bottom', labels: { diff --git a/ietf/static/js/meeting_timeline.js b/ietf/static/js/stats_meeting_timeline.js similarity index 88% rename from ietf/static/js/meeting_timeline.js rename to ietf/static/js/stats_meeting_timeline.js index 713fb3ae707..46b3248209b 100644 --- a/ietf/static/js/meeting_timeline.js +++ b/ietf/static/js/stats_meeting_timeline.js @@ -53,8 +53,14 @@ document.addEventListener('DOMContentLoaded', () => { }, zoom: { zoom: { - wheel: { enabled: true }, // scroll to zoom - pinch: { enabled: true }, // pinch on mobile + wheel: { + enabled: true, + modifierKey: 'alt' // Alt + scroll wheel to zoom + }, // scroll to zoom + pinch: { + enabled: true, + modifierKey: 'alt' + }, // pinch on mobile drag: { // drag to select range enabled: true, modifierKey: 'alt' @@ -63,6 +69,7 @@ document.addEventListener('DOMContentLoaded', () => { }, pan: { enabled: true, + modifierKey: 'alt', mode: 'xy', // pan X-axis and Y-axis }, }, diff --git a/ietf/stats/admin.py b/ietf/stats/admin.py index a29523c19d1..63fccc947ea 100644 --- a/ietf/stats/admin.py +++ b/ietf/stats/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias, MeetingRegistration +from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, AffiliationMainName, CountryAlias, MeetingRegistration class AffiliationAliasAdmin(admin.ModelAdmin): @@ -14,6 +14,11 @@ class AffiliationIgnoredEndingAdmin(admin.ModelAdmin): search_fields = ["ending"] admin.site.register(AffiliationIgnoredEnding, AffiliationIgnoredEndingAdmin) +class AffiliationMainNameAdmin(admin.ModelAdmin): + list_display = ('main_name',) + search_fields = ('main_name',) +admin.site.register(AffiliationMainName, AffiliationMainNameAdmin) + class CountryAliasAdmin(admin.ModelAdmin): list_filter = ["country"] list_display = ["alias", "country"] diff --git a/ietf/stats/factories.py b/ietf/stats/factories.py index 7eba1267528..325c78e209d 100644 --- a/ietf/stats/factories.py +++ b/ietf/stats/factories.py @@ -2,10 +2,24 @@ import factory -from ietf.stats.models import MeetingRegistration +from ietf.stats.models import AffiliationIgnoredEnding, AffiliationMainName, MeetingRegistration from ietf.meeting.factories import MeetingFactory from ietf.person.factories import PersonFactory +class AffiliationIgnoredEndingFactory(factory.django.DjangoModelFactory): + class Meta: + model = AffiliationIgnoredEnding + + ending = 'Inc\\.?' + + +class AffiliationMainNameFactory(factory.django.DjangoModelFactory): + class Meta: + model = AffiliationMainName + + main_name = factory.Faker('company') + + class MeetingRegistrationFactory(factory.django.DjangoModelFactory): class Meta: model = MeetingRegistration diff --git a/ietf/stats/migrations/0003_update_aliases.py b/ietf/stats/migrations/0003_update_aliases.py new file mode 100644 index 00000000000..fa3497c129a --- /dev/null +++ b/ietf/stats/migrations/0003_update_aliases.py @@ -0,0 +1,151 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models + +INITIAL_MAIN_NAMES = ['Adobe', 'Agilent', 'Akamai', 'Alcatel', 'Alcatel-Lucent', 'Alibaba', 'Amazon', 'Apple', 'Arista', 'Aruba', 'AT&T', 'Avaya', + 'BBN', 'Bell Labs', 'Boeing', 'Broadcom', 'Brocade', 'BT', 'Bunyip', 'Cabletron', + 'CERNET', 'Check Point', 'Ciena', 'Cisco', 'Comcast', 'DEC', 'Dell', 'Ericsson', 'EMC', + 'F5', 'Fastmail', 'France Telecom', 'Fraunhofer', 'Fujitsu', + 'Futurewei', 'Google', 'Hewlett-Packard', 'Hitachi', 'HPE', 'Huawei', 'IBM', 'INRIA', 'Intel', 'IEEE', 'ISODE', 'JHU', 'Juniper', + 'KDDI', 'Lucent', 'MCI', 'Meta', 'Microsoft', 'MIT', 'Motorola', 'Mozilla', + 'NASA', 'NEC', 'Netscape','Nokia', 'Nortel', 'NTT', 'NVIDIA', 'Oracle', 'Orange', 'Pantheon', 'Redback', + 'Qualcomm', 'Samsung', 'Siemens', 'SNMP Research', 'Softbank', 'Sun Microsystems', 'SURFnet', + 'Telefonica', 'T-Mobile', 'Telecom Italia', 'Telia', 'Tencent', + 'UUNET', 'VeriSign', 'Verizon', 'Videotron','Vodafone', 'Wellfleet', 'Xerox', 'ZTE'] + +OBSOLETED_AFFILIATION_ALIASES = [ + {'alias': 'cisco systems india pvt', 'name': 'cisco Systems'}, + {'alias': 'cisco systems (india) private limited', 'name': 'cisco Systems'}, + {'alias': 'cisco system', 'name': 'cisco Systems'}, + {'alias': 'cisco', 'name': 'cisco Systems'}, +] + +ADDITIONAL_AFFILIATION_ALIASES = [ + {'alias': 'Asia Pacific Network Information Centre', 'name': 'APNIC'}, + {'alias': 'ATT', 'name': 'AT&T'}, + {'alias': 'AWS', 'name': 'Amazon'}, + {'alias': 'British Telecom', 'name': 'BT'}, + {'alias': 'BUPT', 'name': 'Beijing University of Posts and Telecommunications'}, + {'alias': 'CERT', 'name': 'US-CERT'}, + {'alias': 'CMU', 'name': 'Carnegie Mellon University'}, + {'alias': 'Columbia U.', 'name': 'Columbia University'}, + {'alias': 'Consultant', 'name': 'Independent'}, + {'alias': 'Digital Equipment Corporation', 'name': 'DEC'}, + {'alias': 'HP', 'name': 'Hewlett-Packard'}, + {'alias': 'Independent Consultant', 'name': 'Independent'}, + {'alias': 'Individual', 'name': 'Independent'}, + {'alias': 'Individual Contributor', 'name': 'Independent'}, + {'alias': 'Internet Systems Consortium', 'name': 'ISC'}, + {'alias': 'ISOC', 'name': 'Internet Society'}, + {'alias': 'Johns Hopkins University', 'name': 'JHU'}, + {'alias': 'National Institute of Standards and Technology', 'name': 'US-NIST'}, + {'alias': 'NIST', 'name': 'US-NIST'}, + {'alias': 'Person', 'name': 'Independent'}, + {'alias': 'The Boeing Company', 'name': 'Boeing'}, + {'alias': 'The MITRE Corporation', 'name': 'MITRE'}, + {'alias': 'UCL', 'name': 'University College London'}, + {'alias': 'Unaffiliated', 'name': 'Independent'}, + {'alias': 'Universite catholique de Louvain', 'name': 'UCLouvain'}, + {'alias': 'Université catholique de Louvain', 'name': 'UCLouvain'}, + {'alias': 'University of California, Berkeley', 'name': 'UC Berkeley'}, + {'alias': 'University of California, Los Angeles', 'name': 'UCLA'}, + {'alias': 'US NIST', 'name': 'US-NIST'}, + {'alias': 'USA NIST', 'name': 'US-NIST'}, +] + +ADDITIONAL_IGNORE_ENDINGS = [ + 'ab\\.?', 'ag\\.?', 'corp\\.?', 'corporation\\.?', 'corportation\\.?', 'international pte ltd\\.?', 'limited\\.?', + 'l\\.l\\.c\\.?', + 'private limited\\.?', 'pty ltd\\.?', + 'pvt ltd\\.?', 's\\.a\\.s\\.?', 's\\.a\\.r\\.l\\.?', 's\\.p\\.a\\.?' +] + +NEW_COUNTRY_ALIASES = [ + {'alias': 'belgie', 'country': 'Belgium'}, + {'alias': 'belgique', 'country': 'Belgium'}, + {'alias': 'cccp', 'country': 'Russia'}, + {'alias': 'chinese', 'country': 'China'}, + {'alias': 'finlandia', 'country': 'Finland'}, + {'alias': 'holland', 'country': 'Netherlands'}, + {'alias': 'nederland', 'country': 'Netherlands'}, + {'alias': 'soviet union', 'country': 'Russia'}, + {'alias': 'suomi', 'country': 'Finland'}, + {'alias': 'the netherlands', 'country': 'Netherlands'}, + {'alias': 'u.k.', 'country': 'United Kingdom'}, + {'alias': 'ussr', 'country': 'Russia'}, + {'alias': 'u.s.s.r.', 'country': 'Russia'}, + {'alias': 'россия', 'country': 'Russia'}, + {'alias': 'российская федерация', 'country': 'Russia'}, + {'alias': 'wales', 'country': 'United Kingdom'}, +] + +def forward(apps, schema_editor): + """Add initial main names, update country & affiliation aliases.""" + AffiliationMainName = apps.get_model('stats', 'AffiliationMainName') + for name in INITIAL_MAIN_NAMES: + AffiliationMainName.objects.get_or_create(main_name=name) + + AffiliationAlias = apps.get_model('stats', 'AffiliationAlias') + for entry in OBSOLETED_AFFILIATION_ALIASES: + AffiliationAlias.objects.filter(alias=entry['alias']).delete() + for entry in ADDITIONAL_AFFILIATION_ALIASES: + AffiliationAlias.objects.get_or_create(alias=entry['alias'], defaults={'name': entry['name']}) + + AffiliationIgnoredEnding = apps.get_model('stats', 'AffiliationIgnoredEnding') + for ending in ADDITIONAL_IGNORE_ENDINGS: + AffiliationIgnoredEnding.objects.get_or_create(ending=ending) + + CountryAlias = apps.get_model('stats', 'CountryAlias') + CountryName = apps.get_model('name', 'CountryName') + + for entry in NEW_COUNTRY_ALIASES: + country = CountryName.objects.get(name=entry['country']) + CountryAlias.objects.get_or_create( + alias=entry['alias'], + defaults={'country': country}, + ) + + +def backward(apps, schema_editor): + """Remove initial main names, modified country aliases, and add back some obsolete affiliation aliases.""" + AffiliationMainName = apps.get_model('stats', 'AffiliationMainName') + AffiliationMainName.objects.filter(main_name__in=INITIAL_MAIN_NAMES).delete() + + AffiliationAlias = apps.get_model('stats', 'AffiliationAlias') + for entry in OBSOLETED_AFFILIATION_ALIASES: + AffiliationAlias.objects.get_or_create(alias=entry['alias'], defaults={'name': entry['name']}) + for entry in ADDITIONAL_AFFILIATION_ALIASES: + AffiliationAlias.objects.filter(alias=entry['alias']).delete() + + AffiliationIgnoredEnding = apps.get_model('stats', 'AffiliationIgnoredEnding') + for ending in ADDITIONAL_IGNORE_ENDINGS: + AffiliationIgnoredEnding.objects.filter(ending=ending).delete() + + CountryAlias = apps.get_model('stats', 'CountryAlias') + aliases_to_remove = [entry['alias'] for entry in NEW_COUNTRY_ALIASES] + CountryAlias.objects.filter(alias__in=aliases_to_remove).delete() + +class Migration(migrations.Migration): + + dependencies = [ + ("stats", "0002_fix_meeting_registration_reg_type"), + ] + + operations = [ + migrations.CreateModel( + name='AffiliationMainName', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('main_name', models.CharField(max_length=255, unique=True, help_text="Main leading part of an affiliation will be matched case-insensitive, the remaining part can be ignored for statistical purposes.")), + ], + options={ + 'verbose_name_plural': 'affiliation main names', + }, + ), + migrations.AlterField( + model_name='affiliationignoredending', + name='ending', + field=models.CharField(help_text='Records that ending will be matched case-insensitive and should be stripped from the affiliation for statistical purposes.', max_length=255), + ), + migrations.RunPython(forward, backward), + ] diff --git a/ietf/stats/models.py b/ietf/stats/models.py index 66e359f50ca..2242e82ecea 100644 --- a/ietf/stats/models.py +++ b/ietf/stats/models.py @@ -26,21 +26,33 @@ class AffiliationAlias(models.Model): def __str__(self): return "{} -> {}".format(self.alias, self.name) - def save(self, *args, **kwargs): - self.alias = self.alias.lower() - update_fields = {"alias"}.union(kwargs.pop("update_fields", set())) - super(AffiliationAlias, self).save(update_fields=update_fields, *args, **kwargs) class Meta: verbose_name_plural = "affiliation aliases" + class AffiliationIgnoredEnding(models.Model): """Records that ending should be stripped from the affiliation for statistical purposes.""" - ending = models.CharField(max_length=255, help_text="Regexp with ending, e.g. 'Inc\\.?' - remember to escape .!") + ending = models.CharField(max_length=255, help_text="Records that ending will be matched case-insensitive and should be stripped from the affiliation for statistical purposes.") def __str__(self): - return self.ending + return str(self.ending) + +class AffiliationMainName(models.Model): + """Records that this start of an affiliation is what matters (for statistical purposes).""" + main_name = models.CharField( + max_length=255, + unique=True, + help_text="Main leading part of an affiliation will be matched case-insensitive, the remaining part can be ignored for statistical purposes.") + + class Meta: + verbose_name_plural = 'affiliation main names' + + + def __str__(self): + return str(self.main_name) + class CountryAlias(models.Model): """Records that alias should be treated as country for statistical @@ -55,6 +67,7 @@ def __str__(self): class Meta: verbose_name_plural = "country aliases" + class MeetingRegistration(models.Model): """Registration attendee records from the IETF registration system""" meeting = ForeignKey(Meeting) diff --git a/ietf/stats/resources.py b/ietf/stats/resources.py index 21e2c171acc..3ccd39a6b6c 100644 --- a/ietf/stats/resources.py +++ b/ietf/stats/resources.py @@ -11,7 +11,7 @@ from ietf import api from ietf.api import ToOneField # pyflakes:ignore -from ietf.stats.models import CountryAlias, AffiliationIgnoredEnding, AffiliationAlias, MeetingRegistration +from ietf.stats.models import CountryAlias, AffiliationIgnoredEnding, AffiliationAlias, AffiliationMainName, MeetingRegistration from ietf.name.resources import CountryNameResource @@ -21,7 +21,6 @@ class Meta: queryset = CountryAlias.objects.all() serializer = api.Serializer() cache = SimpleCache() - #resource_name = 'countryalias' ordering = ['id', ] filtering = { "id": ALL, @@ -35,7 +34,6 @@ class Meta: queryset = AffiliationIgnoredEnding.objects.all() serializer = api.Serializer() cache = SimpleCache() - #resource_name = 'affiliationignoredending' ordering = ['id', ] filtering = { "id": ALL, @@ -48,7 +46,6 @@ class Meta: queryset = AffiliationAlias.objects.all() serializer = api.Serializer() cache = SimpleCache() - #resource_name = 'affiliationalias' ordering = ['id', ] filtering = { "id": ALL, @@ -57,6 +54,18 @@ class Meta: } api.stats.register(AffiliationAliasResource()) +class AffiliationMainNameResource(ModelResource): + class Meta: + queryset = AffiliationMainName.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + ordering = ['id', ] + filtering = { + "id": ALL, + "main_name": ALL, + } +api.stats.register(AffiliationMainNameResource()) + from ietf.meeting.resources import MeetingResource from ietf.person.resources import PersonResource class MeetingRegistrationResource(ModelResource): diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 57e3b76b39b..698aa08d72f 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -4,77 +4,506 @@ import datetime import io import json +import re +import factory from django.http import Http404 -from pyquery import PyQuery - -import debug # pyflakes:ignore - from django.test import RequestFactory from django.urls import reverse as urlreverse from django.utils import timezone +from pyquery import PyQuery -from ietf.meeting.models import Meeting -from ietf.utils.test_utils import login_testing_unauthorized, TestCase import ietf.stats.views - - -from ietf.doc.factories import NewRevisionDocEventFactory +import ietf.stats.views_authors +import ietf.stats.views_documents +import ietf.stats.views_meetings +from ietf.doc.factories import ( + DocEventFactory, + DocumentAuthorFactory, + DocumentFactory, + NewRevisionDocEventFactory, + RfcAuthorFactory, + WgDraftFactory, + WgRfcFactory, +) from ietf.group.factories import GroupFactory, RoleFactory -from ietf.person.factories import EmailFactory, PersonFactory -from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory +from ietf.meeting.models import Meeting from ietf.meeting.tests_models import MeetingFactory, RegistrationFactory +from ietf.person.factories import EmailFactory, PersonFactory +from ietf.review.factories import ( + ReviewAssignmentFactory, + ReviewerSettingsFactory, + ReviewRequestFactory, +) +from ietf.stats.factories import ( + AffiliationIgnoredEndingFactory, + AffiliationMainNameFactory, +) from ietf.submit.factories import SubmissionFactory +from ietf.utils.test_utils import TestCase, login_testing_unauthorized class StatisticsTests(TestCase): + def setUp(self): + super().setUp() + llc_staff = GroupFactory(acronym="llc-staff", type_id="team") + self.member = PersonFactory() + RoleFactory(group=llc_staff, name_id="member", person=self.member) + + def _member_login(self): + self.client.login( + username=self.member.user.username, + password=f"{self.member.user.username}+password", + ) + def test_stats_index(self): # Create a meeting as the index page needs to know the current meeting - MeetingFactory(type_id='ietf', number='124', date=timezone.now()) + MeetingFactory(type_id="ietf", number="124", date=timezone.now()) url = urlreverse(ietf.stats.views.stats_index) r = self.client.get(url) + self.assertEqual( + r.status_code, + 200, + msg=f"Unexpected status code {r.status_code} for URL {url}", + ) + + def test_invalid_top_n(self): + url = urlreverse( + ietf.stats.views_authors.authors_timeline, + kwargs={"doc_type": "draft", "stats_type": "country"}, + ) + r = self.client.get(url + "?top=3") self.assertEqual(r.status_code, 200) + self.assertContains(r, "There was an error in your request") + self.assertContains(r, "Invalid top_n choice: 3") + r = self.client.get(url + "?top=eric") + self.assertEqual(r.status_code, 200) + self.assertContains(r, "There was an error in your request") def test_document_stats(self): - # Create a meeting as the index page needs to know the current meeting - MeetingFactory(type_id='ietf', number='124', date=timezone.now()) - r = self.client.get(urlreverse(ietf.stats.views.document_stats)) - self.assertRedirects(r, urlreverse(ietf.stats.views.stats_index)) + timeNow = timezone.now() + yearNow = timeNow.year + time1960 = datetime.datetime( + 1960, 7, 26, 12, 13, 14, tzinfo=datetime.timezone.utc + ) + year1960 = time1960.year + + # Let's create some WGs + group1 = GroupFactory(type_id="wg") + group2 = GroupFactory(type_id="wg") + + # Let's create some RFC and drafts with publication dates + rfcPsGroup1 = WgRfcFactory(std_level_id="ps", group=group1) + DocEventFactory(type="published_rfc", doc=rfcPsGroup1, time=time1960) + rfcExpGroup1 = WgRfcFactory(std_level_id="exp", group=group1) + DocEventFactory(type="published_rfc", doc=rfcExpGroup1, time=time1960) + rfcInfGroup2 = WgRfcFactory(std_level_id="inf", group=group2) + DocEventFactory(type="published_rfc", doc=rfcInfGroup2, time=timeNow) + rfcBcpIAB1 = WgRfcFactory(std_level_id="bcp", stream_id="iab") + DocEventFactory(type="published_rfc", doc=rfcBcpIAB1, time=time1960) + rfcBcpIAB2 = WgRfcFactory(std_level_id="bcp", stream_id="iab") + DocEventFactory(type="published_rfc", doc=rfcBcpIAB2, time=time1960) + wgDraftPsGroup1 = WgDraftFactory( + name="draft-ietf-" + group1.acronym + "-random-thing", + intended_std_level_id="ps", + group=group1, + ) + # Using NewRevisionDocEventFactory(doc=wgDraftPsGroup1, rev="01", time=time1960) does not work + # as pub_date() always use the latest, i.e., more recent "new_revision" event, which is not what we want for this test. + wgDraftPsGroup1.docevent_set.filter(type="new_revision").update(time=time1960) + # The next 2 draft Documents have an auto-created "new_revision" event dated now, which is what we want for this test. + wgDraftPsGroup2 = WgDraftFactory( + name="draft-ietf-" + group2.acronym + "-random-thing", + intended_std_level_id="inf", + group=group2, + ) + # This one has no stream specified and no WG, so it will be counted as "Unspecified" in the stream statistics. + draftExp = DocumentFactory(type_id="draft", intended_std_level_id="exp") + + # Let's create some authors, first get some test strings for affiliations and countries + affiliation = factory.Faker("company").evaluate(None, None, {"locale": None}) + # Sometimes the factory adds "LLC" or some other suffix, causing problem in the tests + # below as another ", LLC" is added. Let's only take the first word of the affiliation + # up to a space or "," + if re.sub(r",?\s*\S+\s*$", "", affiliation) != "": + affiliation = re.sub(r",?\s*\S+\s*$", "", affiliation) + country = factory.Faker("country").evaluate(None, None, {"locale": None}) + # Later tests assume country is not BE/USA + # Let also ensure that the country is a single word to avoid wrongly canonicalised names + # Such as Brunei Darussalam or Lao People's Democratic Republic, which are not canonicalised in the tests below. + while country in {"Belgium", "United States of America"} or " " in country: + country = factory.Faker("country").evaluate(None, None, {"locale": None}) + # Factory sometimes generates country names that are not exactly canonical + # causing problems in the tests below. + if country == "Korea": + country = "South Korea" + + # Create the various aliases ancilliary content + AffiliationIgnoredEndingFactory(ending="llc\\.?") + AffiliationIgnoredEndingFactory(ending="ag\\.?") + AffiliationIgnoredEndingFactory(ending="inc\\.?") + AffiliationIgnoredEndingFactory(ending="corp\\.?") + AffiliationMainNameFactory(main_name="Cisco") + + RfcAuthorFactory(document=rfcPsGroup1, affiliation=affiliation, country=country) + RfcAuthorFactory( + document=rfcExpGroup1, affiliation=affiliation + ", LLC", country=country + ) + RfcAuthorFactory( + document=rfcExpGroup1, + affiliation=factory.Faker("company"), + country=factory.Faker("country"), + ) + DocumentAuthorFactory( + document=wgDraftPsGroup1, affiliation=affiliation + " AG", country=country + ) + RfcAuthorFactory( + document=rfcInfGroup2, affiliation="CiScO InC.", country=country + ) + DocumentAuthorFactory( + document=wgDraftPsGroup2, affiliation="CISCO corp.", country="belgique" + ) + DocumentAuthorFactory( + document=wgDraftPsGroup2, affiliation=affiliation, country=country + ) + RfcAuthorFactory( + document=rfcBcpIAB1, affiliation="CiScO PTY LTD", country="UnItEd StAtEs" + ) + RfcAuthorFactory(document=rfcBcpIAB2, affiliation=affiliation, country="usa") + DocumentAuthorFactory( + document=draftExp, affiliation=affiliation + ",inc", country="U.S.A." + ) + + # Test#1 the documents specific statistics: for RFC about the level + r = self.client.get( + urlreverse( + ietf.stats.views_documents.documents_timeline, + kwargs={"doc_type": "rfc", "stats_type": "level"}, + ) + ) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Specific lines can be removed") + self.assertContains(r, "Rfc Documents by Level") + # Extract the JSON embedded in the response + pq = PyQuery(r.content) + chart_data = json.loads(pq.find("script#chart_data").text()) + self.assertEqual(chart_data["labels"], [year1960, yearNow]) + self.assertTrue( + any( + ds["label"] == "Informational" and ds["data"] == [0, 1] + for ds in chart_data["datasets"] + ), + ) + self.assertTrue( + any( + ds["label"] == "Best Current Practice" and ds["data"] == [2, 0] + for ds in chart_data["datasets"] + ), + ) + + # Test#2 the documents specific statistics: for RFC about the WG + r = self.client.get( + urlreverse( + ietf.stats.views_documents.documents_timeline, + kwargs={"doc_type": "rfc", "stats_type": "wg"}, + ) + ) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Rfc Documents by Wg") + # Extract the JSON embedded in the response + pq = PyQuery(r.content) + chart_data = json.loads(pq.find("script#chart_data").text()) + self.assertEqual(chart_data["labels"], [year1960, yearNow]) + self.assertTrue( + any( + ds["label"] == group1.name and ds["data"] == [2, 0] + for ds in chart_data["datasets"] + ), + ) + + # Test#3 the documents specific statistics: for drafts about the streams + r = self.client.get( + urlreverse( + ietf.stats.views_documents.documents_timeline, + kwargs={"doc_type": "draft", "stats_type": "stream"}, + ) + ) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Draft Documents by Stream") + # Extract the JSON embedded in the response + pq = PyQuery(r.content) + chart_data = json.loads(pq.find("script#chart_data").text()) + self.assertEqual( + chart_data["labels"], [year1960, yearNow], + msg=f"Labels ({chart_data['labels']}) for years do not match expected values=[{year1960}, {yearNow}]", + ) + self.assertTrue( + any( + ds["label"] == "IETF" and ds["data"] == [1, 1] + for ds in chart_data["datasets"] + ), + ) + self.assertTrue( + any( + ds["label"] == "Unspecified" and ds["data"] == [0, 1] + for ds in chart_data["datasets"] + ), + ) + + # Test#4 the authors specific statistics: for all docs about the countries + # With the current production data, there is an error message, so check for it + r = self.client.get( + urlreverse( + ietf.stats.views_authors.authors_timeline, + kwargs={"doc_type": "all", "stats_type": "country"}, + ) + ) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Country information not (yet) available") + # Test#4.bis the authors specific statistics: for all docs about the countries + r = self.client.get( + urlreverse( + ietf.stats.views_authors.authors_timeline, + kwargs={"doc_type": "draft", "stats_type": "country"}, + ) + ) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Draft Authors by Country") + # Extract the JSON embedded in the response + pq = PyQuery(r.content) + chart_data = json.loads(pq.find("script#chart_data").text()) + self.assertEqual( + chart_data["labels"], [year1960, yearNow], + msg=f"Labels ({chart_data['labels']}) for years do not match expected values=[{year1960}, {yearNow}]", + ) + self.assertTrue( + any( + ds["label"] == "United States of America" and ds["data"] == [0, 1] + for ds in chart_data["datasets"] + ), + ) + self.assertTrue( + any( + ds["label"] == "Belgium" and ds["data"] == [0, 1] + for ds in chart_data["datasets"] + ), + ) + self.assertTrue( + any( + ds["label"] == country and ds["data"] == [1, 1] + for ds in chart_data["datasets"] + ), + msg=f"Country '{country}' not found in chart data labels: {chart_data['datasets']}", + ) + + # Test#5 the authors specific statistics: for all all rfcs about the affiliation + r = self.client.get( + urlreverse( + ietf.stats.views_authors.authors_timeline, + kwargs={"doc_type": "rfc", "stats_type": "affiliation"}, + ) + ) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Rfc Authors by Affiliation") + # Extract the JSON embedded in the response + pq = PyQuery(r.content) + chart_data = json.loads(pq.find("script#chart_data").text()) + self.assertEqual(chart_data["labels"], [year1960, yearNow]) + self.assertTrue( + any( + ds["label"].casefold() == affiliation.casefold() + and ds["data"] == [3, 0] + for ds in chart_data["datasets"] + ), + ) + self.assertTrue( + any( + ds["label"] == "Cisco" and ds["data"] == [1, 1] + for ds in chart_data["datasets"] + ), + ) + + # Test#6 the authors specific statistics: for all WG drafts about the country + r = self.client.get( + urlreverse( + ietf.stats.views_authors.authors_timeline, + kwargs={"doc_type": "wg-draft", "stats_type": "country"}, + ) + ) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Wg-Draft Authors by Country") + # Extract the JSON embedded in the response + pq = PyQuery(r.content) + chart_data = json.loads(pq.find("script#chart_data").text()) + self.assertEqual(chart_data["labels"], [year1960, yearNow]) + # Test sometimes failing below with the factory country name being different from the country name in the chart data, + # even though they should be the same country. + # Using casefold to make the comparison more robust, as the country names in the chart data are title-cased + # while the factory can return them in different cases. + self.assertTrue( + any( + ds["label"].casefold() == country.casefold() and ds["data"] == [1, 1] + for ds in chart_data["datasets"] + ), + msg=f"Country '{country}' not found in chart data labels: {chart_data['datasets']}", + ) + self.assertTrue( + any( + ds["label"] == "Belgium" and ds["data"] == [0, 1] + for ds in chart_data["datasets"] + ), + ) + + # Test#7 the authors specific statistics global + r = self.client.get( + urlreverse( + ietf.stats.views_authors.authors_total, + kwargs={"doc_type": "draft", "stats_type": "country"}, + ) + ) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Draft Authors by Country") + # Extract the JSON embedded in the response + pq = PyQuery(r.content) + chart_data = json.loads(pq.find("script#chart_data").text()) + self.assertTrue("Belgium" in chart_data["labels"]) + self.assertTrue("United States of America" in chart_data["labels"]) + self.assertTrue(country in chart_data["labels"]) + USA_index = chart_data["labels"].index("United States of America") + # Let's check whether USA has indeed 1 + self.assertEqual(chart_data["datasets"][0]["data"][USA_index], 1) + + # Test#8 the documents specific statistics global + r = self.client.get( + urlreverse( + ietf.stats.views_documents.documents_total, + kwargs={"doc_type": "draft", "stats_type": "wg"}, + ) + ) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Draft Documents by Wg") + # Extract the JSON embedded in the response + pq = PyQuery(r.content) + chart_data = json.loads(pq.find("script#chart_data").text()) + self.assertTrue(group1.name in chart_data["labels"]) + individual_index = chart_data["labels"].index("Individual submissions") + # Let's check whether USA has indeed 1 + self.assertEqual(chart_data["datasets"][0]["data"][individual_index], 1) + + # Test#10 Check the used affiliations list view w/o being logged in, which should redirect to the login page + r = self.client.get(urlreverse(ietf.stats.views.used_affiliations_list)) + self.assertEqual(r.status_code, 302) + self.assertIn("/accounts/login", r["Location"]) + + # Test#10-bis, check the used affiliations list view while being logged in as a non-LLC staff member, which should return 403 + self._member_login() + r = self.client.get(urlreverse(ietf.stats.views.used_affiliations_list)) + self.assertEqual(r.status_code, 200) + self.assertTrue( + any( + [cell.text for cell in row.findall("td")] + == ["CISCO corp.", "1", "Cisco"] + for row in PyQuery(r.content)("tr") + ) + ) def test_meeting_stats(self): - meeting124 = MeetingFactory(type_id='ietf', number='124', date=timezone.now()) - meeting125 = MeetingFactory(type_id='ietf', number='125', date=timezone.now() + datetime.timedelta(days=120)) - RegistrationFactory.create_batch(15, meeting=meeting124, with_ticket={'attendance_type_id': 'onsite'}, attended=True) - RegistrationFactory(meeting=meeting124, with_ticket={'attendance_type_id': 'onsite'}, attended=False) - RegistrationFactory.create_batch(14, meeting=meeting124, with_ticket={'attendance_type_id': 'remote'}, attended=True) - RegistrationFactory(meeting=meeting124, with_ticket={'attendance_type_id': 'remote'}, attended=False) - RegistrationFactory.create_batch(15, meeting=meeting125, affiliation='Test LLC', with_ticket={'attendance_type_id': 'remote'}, attended=False) - RegistrationFactory.create_batch(25, meeting=meeting125, affiliation='Example, Ltd', with_ticket={'attendance_type_id': 'onsite'}, attended=False) + meeting124 = MeetingFactory(type_id="ietf", number="124", date=timezone.now()) + meeting125 = MeetingFactory( + type_id="ietf", + number="125", + date=timezone.now() + datetime.timedelta(days=120), + ) + RegistrationFactory.create_batch( + 15, + meeting=meeting124, + with_ticket={"attendance_type_id": "onsite"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting124, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + ) + RegistrationFactory.create_batch( + 14, + meeting=meeting124, + with_ticket={"attendance_type_id": "remote"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting124, + with_ticket={"attendance_type_id": "remote"}, + attended=False, + ) + RegistrationFactory.create_batch( + 15, + meeting=meeting125, + affiliation="Test LLC", + with_ticket={"attendance_type_id": "remote"}, + attended=False, + ) + RegistrationFactory.create_batch( + 25, + meeting=meeting125, + affiliation="Example, Ltd", + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + ) + + # Create the various aliases ancilliary content + AffiliationIgnoredEndingFactory(ending="llc\\.?") + AffiliationIgnoredEndingFactory(ending="ltd\\.?") + # Test the meeting specific statitistics per affiliation and per country - r = self.client.get(urlreverse(ietf.stats.views.meeting_stats, kwargs={"meeting_number": "124", "stats_type": "affiliation"})) + r = self.client.get( + urlreverse( + ietf.stats.views_meetings.meeting_stats, + kwargs={"meeting_number": "124", "stats_type": "affiliation"}, + ) + ) self.assertEqual(r.status_code, 200) self.assertContains(r, "Total Registrations by Affiliation (31 in total)") self.assertContains(r, "In Person Registrations by Affiliation (16 in total)") - self.assertContains(r, "/stats/meeting/124/affiliation") - self.assertContains(r, "/stats/meeting/125/affiliation") - r = self.client.get(urlreverse(ietf.stats.views.meeting_stats, kwargs={"meeting_number": "124", "stats_type": "country"})) + self.assertContains(r, "/stats/meetings/124/affiliation") + self.assertContains(r, "/stats/meetings/125/affiliation") + r = self.client.get( + urlreverse( + ietf.stats.views_meetings.meeting_stats, + kwargs={"meeting_number": "124", "stats_type": "country"}, + ) + ) self.assertEqual(r.status_code, 200) self.assertContains(r, "Total Registrations by Country (31 in total)") self.assertContains(r, "In Person Registrations by Country (16 in total)") - self.assertContains(r, "/stats/meeting/124/country") - self.assertContains(r, "/stats/meeting/125/country") + self.assertContains(r, "/stats/meetings/124/country") + self.assertContains(r, "/stats/meetings/125/country") # Test the meetings timeline per country - r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "country"})) + r = self.client.get( + urlreverse( + ietf.stats.views_meetings.meetings_timeline, + kwargs={"stats_type": "country"}, + ) + ) self.assertEqual(r.status_code, 200) - self.assertContains(r, "/stats/meeting/124/country") - self.assertContains(r, "/stats/meeting/125/country") - self.assertContains(r, "This page provides a timeline of meeting registrations by country") + self.assertContains(r, "/stats/meetings/124/country") + self.assertContains(r, "/stats/meetings/125/country") + self.assertContains( + r, "This page provides a timeline of meeting registrations by country" + ) # Test the meetings timeline per affiliation - r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "affiliation"})) + r = self.client.get( + urlreverse( + ietf.stats.views_meetings.meetings_timeline, + kwargs={"stats_type": "affiliation"}, + ) + ) self.assertEqual(r.status_code, 200) - self.assertContains(r, "/stats/meeting/124/affiliation") - self.assertContains(r, "/stats/meeting/125/affiliation") - self.assertContains(r, "This page provides a timeline of meeting registrations by affiliation") + self.assertContains(r, "/stats/meetings/124/affiliation") + self.assertContains(r, "/stats/meetings/125/affiliation") + self.assertContains( + r, "This page provides a timeline of meeting registrations by affiliation" + ) # Extract the JSON embedded in the response pq = PyQuery(r.content) in_person_data = json.loads(pq.find("script#in-person-chart-data").text()) @@ -82,23 +511,60 @@ def test_meeting_stats(self): any( ds["label"] == "Example" and ds["data"] == [0, 25] for ds in in_person_data["datasets"] + ), + ) + # Test for CSV download of the meeting stats per affiliation + r = self.client.get( + urlreverse( + ietf.stats.views_meetings.meeting_stats, + kwargs={"meeting_number": "125", "stats_type": "affiliation"}, ) + + "?download=total&top=5" ) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "text/csv") + self.assertIn("attachment;", r["Content-Disposition"]) + csv_lines = r.content.decode("utf-8").splitlines() + self.assertTrue(csv_lines[0].startswith('"affiliation","count"')) + self.assertTrue(csv_lines[1].startswith('"Example",25')) # Test the global meetings timeline - r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "total"})) + r = self.client.get( + urlreverse( + ietf.stats.views_meetings.meetings_timeline, + kwargs={"stats_type": "reg_type"}, + ) + ) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meetings/124/country") + self.assertContains(r, "/stats/meetings/125/country") + self.assertContains( + r, "This page provides a timeline of meeting registrations." + ) + # Test for CSV download of the meetings timeline per affiliation + r = self.client.get( + urlreverse( + ietf.stats.views_meetings.meetings_timeline, + kwargs={"stats_type": "affiliation"}, + ) + + "?download=total&top=5" + ) self.assertEqual(r.status_code, 200) - self.assertContains(r, "/stats/meeting/124/country") - self.assertContains(r, "/stats/meeting/125/country") - self.assertContains(r, "This page provides a timeline of meeting registrations.") + self.assertEqual(r["Content-Type"], "text/csv") + self.assertIn("attachment;", r["Content-Disposition"]) + body = r.content.decode("utf-8") + csv_lines = body.splitlines() + self.assertTrue(csv_lines[0].startswith('"IETF meeting","affiliation","count"')) + self.assertTrue(csv_lines[1].startswith('124,"Example",0')) + self.assertIn('125,"Test",15', body) def test_meeting_stats_for_bad_meeting(self): self.assertFalse(Meeting.objects.filter(number=676767).exists()) for stats_type in ["affiliation", "country"]: r = self.client.get( urlreverse( - "ietf.stats.views.meeting_stats", + "ietf.stats.views_meetings.meeting_stats", kwargs={"meeting_number": 676767, "stats_type": stats_type}, - ) + ), ) self.assertEqual(r.status_code, 404) @@ -107,34 +573,120 @@ def test_meeting_stats_for_bad_meeting(self): interim_num = MeetingFactory(type_id="interim").number request_factory = RequestFactory() with self.assertRaises(Http404): - ietf.stats.views.meeting_stats( - request_factory.get(f"/stats/meeting/{interim_num}/{stats_type}"), + ietf.stats.views_meetings.meeting_stats( + request_factory.get(f"/stats/meetings/{interim_num}/{stats_type}"), meeting_number=interim_num, stats_type=stats_type, ) - def test_known_country_list(self): - # check redirect + +class LegacyUrlRedirectTests(TestCase): + """The old /stats/document/... and /stats/meeting/... URLs should permanently + redirect to their new /stats/documents/... and /stats/meetings/... equivalents.""" + + def test_document_timeline_redirect(self): + for doc_type in ["draft", "rfc"]: + for stats_type in ["level", "stream", "wg"]: + old_url = f"/stats/document/{doc_type}/{stats_type}/" + new_url = urlreverse( + ietf.stats.views_documents.documents_timeline, + kwargs={"doc_type": doc_type, "stats_type": stats_type}, + ) + r = self.client.get(old_url) + self.assertRedirects( + r, new_url, status_code=301, fetch_redirect_response=False + ) + + def test_meeting_timeline_redirect(self): + new_url = urlreverse(ietf.stats.views_meetings.meetings_timeline) + r = self.client.get("/stats/meeting/") + self.assertRedirects( + r, new_url, status_code=301, fetch_redirect_response=False + ) + + for stats_type in ["affiliation", "country"]: + new_url = urlreverse( + ietf.stats.views_meetings.meetings_timeline, + kwargs={"stats_type": stats_type}, + ) + r = self.client.get(f"/stats/meeting/{stats_type}/") + self.assertRedirects( + r, new_url, status_code=301, fetch_redirect_response=False + ) + + # "total" was renamed to "reg_type" + new_url = urlreverse( + ietf.stats.views_meetings.meetings_timeline, + kwargs={"stats_type": "reg_type"}, + ) + r = self.client.get("/stats/meeting/total/") + self.assertRedirects( + r, new_url, status_code=301, fetch_redirect_response=False + ) + + def test_meeting_stats_redirect(self): + meeting = MeetingFactory(type_id="ietf") + for stats_type in ["affiliation", "country"]: + old_url = f"/stats/meeting/{meeting.number}/{stats_type}/" + new_url = urlreverse( + ietf.stats.views_meetings.meeting_stats, + kwargs={"meeting_number": meeting.number, "stats_type": stats_type}, + ) + r = self.client.get(old_url) + self.assertRedirects( + r, new_url, status_code=301, fetch_redirect_response=False + ) + + +class KnownCountriesTests(TestCase): + def setUp(self): + super().setUp() + llc_staff = GroupFactory(acronym="llc-staff", type_id="team") + self.member = PersonFactory() + RoleFactory(group=llc_staff, name_id="member", person=self.member) + self.non_member = PersonFactory() + + def _member_login(self): + self.client.login( + username=self.member.user.username, + password=f"{self.member.user.username}+password", + ) + + def test_access_unauthenticated(self): url = urlreverse(ietf.stats.views.known_countries_list) + r = self.client.get(url) + self.assertEqual(r.status_code, 302) + self.assertIn("/accounts/login", r["Location"]) + + def test_known_countries_list(self): + url = urlreverse(ietf.stats.views.known_countries_list) + self._member_login() r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertContains(r, "United States") +class ReviewStatsTests(TestCase): def test_review_stats(self): reviewer = PersonFactory() - review_req = ReviewRequestFactory(state_id='assigned') - ReviewAssignmentFactory(review_request=review_req, state_id='assigned', reviewer=reviewer.email_set.first()) - RoleFactory(group=review_req.team,name_id='reviewer',person=reviewer) + review_req = ReviewRequestFactory(state_id="assigned") + ReviewAssignmentFactory( + review_request=review_req, + state_id="assigned", + reviewer=reviewer.email_set.first(), + ) + RoleFactory(group=review_req.team, name_id="reviewer", person=reviewer) ReviewerSettingsFactory(team=review_req.team, person=reviewer) - PersonFactory(user__username='plain') + PersonFactory(user__username="plain") # check redirect - url = urlreverse(ietf.stats.views.review_stats) + url = urlreverse(ietf.stats.views_reviews.review_stats) login_testing_unauthorized(self, "secretary", url) - completion_url = urlreverse(ietf.stats.views.review_stats, kwargs={ "stats_type": "completion" }) + completion_url = urlreverse( + ietf.stats.views_reviews.review_stats, kwargs={"stats_type": "completion"} + ) r = self.client.get(url) self.assertEqual(r.status_code, 302) @@ -148,7 +700,9 @@ def test_review_stats(self): # check tabular self.client.login(username="secretary", password="secretary+password") for stats_type in ["completion", "results", "states"]: - url = urlreverse(ietf.stats.views.review_stats, kwargs={ "stats_type": stats_type }) + url = urlreverse( + ietf.stats.views_reviews.review_stats, kwargs={"stats_type": stats_type} + ) r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -156,38 +710,48 @@ def test_review_stats(self): self.assertTrue(q('.review-stats td:contains("1")')) # check stacked chart - url = urlreverse(ietf.stats.views.review_stats, kwargs={ "stats_type": "time" }) - url += "?team={}".format(review_req.team.acronym) + + url = urlreverse( + ietf.stats.views_reviews.review_stats, kwargs={"stats_type": "time"} + ) + url += f"?team={review_req.team.acronym}" r = self.client.get(url) self.assertEqual(r.status_code, 200) - stacked_data = json.loads(r.context['data']) + stacked_data = json.loads(r.context["data"]) # Ignore the timestamp elements, just check that the data is correct self.assertEqual(len(stacked_data), 2) - self.assertEqual(stacked_data[0]['label'], 'in time') - self.assertEqual(stacked_data[0]['color'], '#3d22b3') - self.assertEqual(stacked_data[0]['data'], [[stacked_data[0]['data'][0][0], 0]]) - self.assertEqual(stacked_data[1]['label'], 'late') - self.assertEqual(stacked_data[1]['color'], '#b42222') - self.assertEqual(stacked_data[1]['data'], [[stacked_data[0]['data'][0][0], 0]]) + self.assertEqual(stacked_data[0]["label"], "in time") + self.assertEqual(stacked_data[0]["color"], "#3d22b3") + self.assertEqual(stacked_data[0]["data"], [[stacked_data[0]["data"][0][0], 0]]) + self.assertEqual(stacked_data[1]["label"], "late") + self.assertEqual(stacked_data[1]["color"], "#b42222") + self.assertEqual(stacked_data[1]["data"], [[stacked_data[0]["data"][0][0], 0]]) q = PyQuery(r.content) - self.assertTrue(q('#stats-time-graph')) + self.assertTrue(q("#stats-time-graph")) # check non-stacked chart - url = urlreverse(ietf.stats.views.review_stats, kwargs={ "stats_type": "time" }) - url += "?team={}".format(review_req.team.acronym) + url = urlreverse( + ietf.stats.views_reviews.review_stats, kwargs={"stats_type": "time"} + ) + url += f"?team={review_req.team.acronym}" url += "&completion=not_completed" r = self.client.get(url) self.assertEqual(r.status_code, 200) - non_stacked_data = json.loads(r.context['data']) + non_stacked_data = json.loads(r.context["data"]) # Ignore the timestamp elements, just check that the data is correct self.assertEqual(len(non_stacked_data), 1) - self.assertEqual(non_stacked_data[0]['color'], '#3d22b3') - self.assertEqual(non_stacked_data[0]['data'], [[non_stacked_data[0]['data'][0][0], 0]]) + self.assertEqual(non_stacked_data[0]["color"], "#3d22b3") + self.assertEqual( + non_stacked_data[0]["data"], [[non_stacked_data[0]["data"][0][0], 0]] + ) q = PyQuery(r.content) - self.assertTrue(q('#stats-time-graph')) + self.assertTrue(q("#stats-time-graph")) # check reviewer level - url = urlreverse(ietf.stats.views.review_stats, kwargs={ "stats_type": "completion", "acronym": review_req.team.acronym }) + url = urlreverse( + ietf.stats.views_reviews.review_stats, + kwargs={"stats_type": "completion", "acronym": review_req.team.acronym}, + ) r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -256,8 +820,20 @@ def test_summary_counts(self): submitter_email="submitter@example.com", ) sub.authors = [ - {"name": "Author One", "email": "author1@example.com", "affiliation": "", "country": "", "errors": []}, - {"name": "Author Two", "email": "author2@example.com", "affiliation": "", "country": "", "errors": []}, + { + "name": "Author One", + "email": "author1@example.com", + "affiliation": "", + "country": "", + "errors": [], + }, + { + "name": "Author Two", + "email": "author2@example.com", + "affiliation": "", + "country": "", + "errors": [], + }, ] sub.save() NewRevisionDocEventFactory( @@ -278,7 +854,9 @@ def test_summary_counts(self): doc=extra.doc, ) - url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + url = urlreverse( + ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)} + ) r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertEqual(r.context["year"], year) @@ -298,11 +876,19 @@ def test_download_authors_csv(self): submission_date=datetime.date(year, 4, 1), ) sub.authors = [ - {"name": "Author", "email": "csvauthor@example.com", "affiliation": "", "country": "", "errors": []}, + { + "name": "Author", + "email": "csvauthor@example.com", + "affiliation": "", + "country": "", + "errors": [], + }, ] sub.save() - url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + url = urlreverse( + ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)} + ) r = self.client.get(url, {"download": "authors"}) self.assertEqual(r.status_code, 200) self.assertEqual(r["Content-Type"], "text/csv") @@ -320,7 +906,9 @@ def test_download_submitters_csv(self): submitter_email="csvsubmitter@example.com", ) - url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + url = urlreverse( + ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)} + ) r = self.client.get(url, {"download": "submitters"}) self.assertEqual(r.status_code, 200) self.assertEqual(r["Content-Type"], "text/csv") diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index 3bb107813a6..2bc5dd2da03 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -3,17 +3,49 @@ from django.conf import settings +from django.shortcuts import redirect +from django.urls import reverse as urlreverse from ietf.stats import views from ietf.utils.urls import url +from ietf.stats import views_authors, views_documents, views_meetings, views_reviews + +# "total" was renamed to "reg_type" when meetings_timeline moved to views_meetings.py +_OLD_MEETING_STATS_TYPE_MAP = {"total": "reg_type"} + +def _redirect_old_meetings_timeline(request, stats_type=None): + kwargs = {} + if stats_type is not None: + kwargs["stats_type"] = _OLD_MEETING_STATS_TYPE_MAP.get(stats_type, stats_type) + return redirect(urlreverse(views_meetings.meetings_timeline, kwargs=kwargs), permanent=True) + +def _redirect_old_documents_timeline(request, doc_type, stats_type): + return redirect( + urlreverse(views_documents.documents_timeline, kwargs={"doc_type": doc_type, "stats_type": stats_type}), + permanent=True, + ) + +def _redirect_old_meeting_stats(request, meeting_number, stats_type): + return redirect( + urlreverse(views_meetings.meeting_stats, kwargs={"meeting_number": meeting_number, "stats_type": stats_type}), + permanent=True, + ) + +# Some URLs have changed during the development, so we need to redirect the old ones to the new ones. urlpatterns = [ url(r"^$", views.stats_index), - url(r"^document/(?:(?Pauthors|pages|words|format|formlang|author/(?:documents|affiliation|country|continent|citations|hindex)|yearly/(?:affiliation|country|continent))/)?$", views.document_stats), - url(r"^knowncountries/$", views.known_countries_list), - url(r"^meeting/$", views.meetings_timeline), - url(r"^meeting/(?P\d+)/(?Paffiliation|country)/$", views.meeting_stats), - url(r"^meeting/(?:(?Paffiliation|country|total)/)?$", views.meetings_timeline), - url(r"^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), url(r"^annual_report_inputs/(?:(?P\d{4})/)?$", views.annual_report_inputs), + url(r"^authors/(?Pall|draft|wg-draft|rfc)/(?Paffiliation|country)/total/$", views_authors.authors_total), + url(r"^authors/(?Pall|draft|wg-draft|rfc)/(?Paffiliation|country)/$", views_authors.authors_timeline), + url(r"^document/(?Pdraft|rfc)/(?Plevel|stream|wg)/$", _redirect_old_documents_timeline), + url(r"^documents/(?Pdraft|rfc)/(?Plevel|stream|wg)/total/$", views_documents.documents_total), + url(r"^documents/(?Pdraft|rfc)/(?Plevel|stream|wg)/$", views_documents.documents_timeline), + url(r"^knowncountries/$", views.known_countries_list), + url(r"^meeting/(?P\d+)/(?Paffiliation|country)/$", _redirect_old_meeting_stats), + url(r"^meeting/(?:(?Paffiliation|country|total)/)?$", _redirect_old_meetings_timeline), + url(r"^meetings/(?:(?Paffiliation|country|reg_type)/)?$", views_meetings.meetings_timeline), + url(r"^meetings/(?P\d+)/(?Paffiliation|country)/$", views_meetings.meeting_stats), + url(r"^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views_reviews.review_stats), + url(r"^usedaffiliations/$", views.used_affiliations_list), ] diff --git a/ietf/stats/utils.py b/ietf/stats/utils.py index a13e87a4f47..962b450f413 100644 --- a/ietf/stats/utils.py +++ b/ietf/stats/utils.py @@ -3,20 +3,69 @@ import re -from collections import defaultdict +import hashlib import debug # pyflakes:ignore -from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias +from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, AffiliationMainName, CountryAlias from ietf.name.models import CountryName import logging logger = logging.getLogger('django') - +# Color palette for lines +colors = [ + '#FF6384', '#36A2EB', '#FFCE56', '#4BC0C0', '#9966FF', + '#FF9F40', '#C9CBCF', '#7BC043', '#F37735', '#00ABA9', + '#2B5797', '#E81123', '#00A4EF', '#7FBA00', '#FFB900', + '#D83B01', '#B4009E', '#5C2D91', '#008575', '#E3008C', + # New additions — same vibrant/saturated theme + '#A4C639', '#FF7043', '#26A69A', '#AB47BC', '#42A5F5', + '#EC407A', '#FFA726', '#66BB6A', '#5E35B1', '#29B6F6', + '#D4AC0D', '#8E44AD', '#16A085', '#C0392B', '#2980B9', + '#E67E22', '#27AE60', '#CB4335', '#1F618D', '#AF7AC5', +] + +def color_from_hash(s): + if s == 'Unspecified': + return "#B0B0B0" + if s == 'Other': + return "#E0E0E0" + full_hash = hashlib.md5(s.encode('utf-8'), usedforsecurity=False).digest() + hash = int.from_bytes(full_hash[:2]) + return colors[hash % len(colors)] + +top_n_choices = [5, 10, 20, 50, 100] + +def get_top_n_choices(): + return top_n_choices + +def get_valid_top_n(request): + """Return a valid top-N value or the corresponding error response.""" + from django.shortcuts import render + + try: + top_n = int(request.GET.get("top", "20")) + except ValueError: + return 0, render( + request, + "stats/error.html", + {"message": f"Invalid top_n choice: {request.GET.get('top')}. Valid choices are: {get_top_n_choices()}"}, + ) + + if top_n not in top_n_choices: + return 0, render( + request, + "stats/error.html", + {"message": f"Invalid top_n choice: {top_n}. Valid choices are: {get_top_n_choices()}"}, + ) + + return top_n, None + def compile_affiliation_ending_stripping_regexp(): parts = [] for ending_re in AffiliationIgnoredEnding.objects.values_list("ending", flat=True): + # Try to compile as a syntax check try: re.compile(ending_re) except re.error: @@ -24,7 +73,9 @@ def compile_affiliation_ending_stripping_regexp(): parts.append(ending_re) - re_str = ",? *({}) *$".format("|".join(parts)) + # Build a regexp that matches any of the endings at the end of a string, + # optionally preceded by whitespace or commas. The regexp is case-insensitive. + re_str = "(?:, *| +)({}) *$".format("|".join(parts)) return re.compile(re_str, re.IGNORECASE) @@ -36,27 +87,30 @@ def get_aliased_affiliations(affiliations): We employ the following strategies, interleaved: - Stripping company endings like Inc., GmbH etc. from database - + - Using a leading name from the database, like "Google Analytics" -> "Google" - Looking up aliases stored directly in the database, like - "Examplar International" -> "Examplar" - - - Case-folding so Examplar and EXAMPLAR is merged with the - winner being the one with most occurrences (so input should not - be made unique) or most upper case letters in case of ties. - Case folding can be overridden by the aliases in the database.""" + "Examplar International" -> "Examplar" """ res = {} ending_re = compile_affiliation_ending_stripping_regexp() known_aliases = { alias.lower(): name for alias, name in AffiliationAlias.objects.values_list("alias", "name") } + # Let's prepare a dict for things like "Google Inc." or "Google Analytics"-> "Google" + # by adding a single space to the end of the main name + # so we only match it at the beginning of the affiliation and not in the middle of it, e.g. "Google Analytics" will match "Google" + # but neither "My Google Analytics" nor "GoogleIsGreat" will match "Google" + # sort longest-first so that when one main name is a prefix of another, the more specific (longer) one wins + affiliation_main_names = sorted( + ((main_name.strip(" ").lower() + ' ', main_name) for main_name in AffiliationMainName.objects.values_list("main_name", flat=True)), + key=lambda pair: len(pair[0]), + reverse=True, + ) - affiliations_with_case_spellings = defaultdict(set) - case_spelling_count = defaultdict(int) for affiliation in affiliations: original_affiliation = affiliation - # check aliases from DB + # check aliases from Aliases DB name = known_aliases.get(affiliation.lower()) if name is not None: affiliation = name @@ -68,28 +122,18 @@ def get_aliased_affiliations(affiliations): affiliation = name res[original_affiliation] = affiliation - # check aliases from DB + # check again aliases from Aliases DB after stripping the ending name = known_aliases.get(affiliation.lower()) if name is not None: affiliation = name res[original_affiliation] = affiliation - affiliations_with_case_spellings[affiliation.lower()].add(original_affiliation) - case_spelling_count[affiliation] += 1 - - def affiliation_sort_key(affiliation): - count = case_spelling_count[affiliation] - uppercase_letters = sum(1 for c in affiliation if c.isupper()) - return (count, uppercase_letters) - - # now we just need to pick the most popular uppercase/lowercase - # spelling for each affiliation with more than one - for similar_affiliations in affiliations_with_case_spellings.values(): - if len(similar_affiliations) > 1: - most_popular = sorted(similar_affiliations, key=affiliation_sort_key, reverse=True)[0] - for affiliation in similar_affiliations: - if affiliation != most_popular: - res[affiliation] = most_popular + # check aliases from Main Names DB + affiliation_plus_space = affiliation.strip(" ") + " " # to match main names with a single space added to the end of them + name = next((original for lower, original in affiliation_main_names if affiliation.lower().startswith(lower) or affiliation_plus_space.lower().startswith(lower)), None) + if name is not None: + affiliation = name + res[original_affiliation] = affiliation return res diff --git a/ietf/stats/views.py b/ietf/stats/views.py index fe2fa82f554..19e818fd57e 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -1,129 +1,62 @@ # Copyright The IETF Trust 2016-2026, All Rights Reserved -# -*- coding: utf-8 -*- -import calendar import csv import datetime -import itertools -import json -import dateutil.relativedelta -from collections import defaultdict -from django.conf import settings -from django.contrib.auth.decorators import login_required -from django.core.cache import cache from django.http import HttpResponse, HttpResponseRedirect -from django.shortcuts import render, get_object_or_404 +from django.shortcuts import render from django.urls import reverse as urlreverse from django.db.models import Count -import debug # pyflakes:ignore - -from ietf.review.utils import (extract_review_assignment_data, - aggregate_raw_period_review_assignment_stats, - ReviewAssignmentData, - sum_period_review_assignment_stats, - sum_raw_review_assignment_aggregations) -from ietf.group.models import Role, Group -from ietf.person.models import Person -from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName -from ietf.meeting.models import Registration, Meeting -from ietf.ietfauth.utils import has_role, role_required -from ietf.utils.response import permission_denied -from ietf.utils.timezone import date_today, DEADLINE_TZINFO +from ietf.ietfauth.utils import role_required from ietf.meeting.helpers import get_current_ietf_meeting_num +from ietf.name.models import CountryName +from ietf.doc.models import DocumentAuthor +from ietf.stats.utils import get_aliased_affiliations -# Color palette for lines -colors = [ - '#FF6384', '#36A2EB', '#FFCE56', '#4BC0C0', '#9966FF', - '#FF9F40', '#C9CBCF', '#7BC043', '#F37735', '#00ABA9', - '#2B5797', '#E81123', '#00A4EF', '#7FBA00', '#FFB900', - '#D83B01', '#B4009E', '#5C2D91', '#008575', '#E3008C', -] def stats_index(request): """Render the statistics index page with the current meeting number as it is required by the meeting menu item.""" current_meeting = get_current_ietf_meeting_num() return render(request, "stats/index.html", { - "current_meeting": current_meeting + "current_meeting": current_meeting, }) -def generate_query_string(query_dict, overrides): - """ - Returns: - A query string starting with '?' if there are parameters, empty string otherwise. - """ - query_part = "" +@role_required("LLC Staff") +def used_affiliations_list(request): + """Render a list of used affiliations in the DocAuthor model with their aliases.""" + qs = ( + DocumentAuthor.objects + .filter(document__type_id="draft") + .values('affiliation') + .annotate(author_count=Count("person", distinct=True)) + ) - if query_dict or overrides: - d = query_dict.copy() - for k, v in overrides.items(): - if type(v) in (list, tuple): - if not v: - if k in d: - del d[k] - else: - d.setlist(k, v) + affiliations = [] + for row in qs: + affiliation = row['affiliation'] + author_count = row['author_count'] + aliases_map = get_aliased_affiliations([affiliation]) + if affiliation in aliases_map: + if aliases_map[affiliation] != affiliation: + canonical = aliases_map[affiliation] else: - if v is None or v == "": - if k in d: - del d[k] - else: - d[k] = v - - if d: - query_part = "?" + d.urlencode() - - return query_part - -def get_choice(request, get_parameter, possible_choices, multiple=False): - """Extract a choice from the request GET parameters. - - Since statistics pages use links for navigation instead of forms, - this helper selects between possible choices from the URL parameters. - - Args: - request: The HTTP request object. - get_parameter: The name of the GET parameter. - possible_choices: List of tuples (value, label). - multiple: If True, return a list of found values; otherwise return the first found or None. - - Returns: - The selected value(s) or None. - """ - values = request.GET.getlist(get_parameter) - found = [t[0] for t in possible_choices if t[0] in values] - - if multiple: - return found - else: - if found: - return found[0] + canonical = '' # affiliation is already canonical else: - return None - -def add_url_to_choices(choices, url_builder): - """Add URLs to a list of choices. - - Args: - choices: List of tuples (slug, label). - url_builder: Function that takes a slug and returns a URL. - - Returns: - List of tuples (slug, label, url). - """ - return [ (slug, label, url_builder(slug)) for slug, label in choices] + canonical = '' # Nothing was found, affiliation is assumed to be canonical + affiliations.append({ + "affiliation": affiliation, + "author_count": author_count, + "canonical": canonical, + }) -def document_stats(request, stats_type=None): - # timeline per year, or per specific year: streams, affiliation, rfc vs I-D - # could also be time between individual/WG I-D to rfc publication/IESG ballot - # DISCUSS resolution time - # Humm also split by authors (affiliation) / documents (the rest) probably - """Redirect to the stats index page. Deprecated view.""" - return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + return render(request, "stats/used_affiliations_list.html", { + "affiliations": affiliations, + }) -def known_countries_list(request, stats_type=None, acronym=None): +@role_required("LLC Staff") +def known_countries_list(request): """Render a list of known countries with their aliases.""" countries = CountryName.objects.prefetch_related("countryalias_set") for c in countries: @@ -135,836 +68,11 @@ def known_countries_list(request, stats_type=None, acronym=None): "countries": countries, }) -def canonicalize_affiliation(affiliation): - """Canonicalize an affiliation string by removing common suffixes and standardizing prefixes. - - Args: - affiliation: The affiliation string to canonicalize. - - Returns: - The canonicalized affiliation string, or None if input is None. - """ - if not affiliation or affiliation.lower() in ('n/a', 'none', 'unspecified'): - return None - for suffix in ('ab', 'ag', 'corp', 'corp.', 'corporation', 'gmbh', 'inc.', 'inc', 'international pte ltd', 'llc', 'ltd', 'ltd.', 'private limited', 'pty ltd', 'pvt ltd'): - if affiliation.lower().endswith(', ' + suffix): - affiliation = affiliation[:-(len(suffix)+2)] - elif affiliation.lower().endswith(' ' + suffix): - affiliation = affiliation[:-(len(suffix)+1)] - elif affiliation.lower().endswith(',' + suffix): - affiliation = affiliation[:-(len(suffix)+1)] - for prefix in ('akamai','apple', 'cisco', 'futurewei', 'google', 'hitachi', 'hpe', 'huawei', 'juniper', 'meta', 'nokia', 'ntt', 'siemens'): - if affiliation.lower().startswith(prefix + ' '): - affiliation = prefix - return affiliation.title() - -def get_affiliation_data_for_meetings(attendance_type=None): - """Get affiliation participation data for meetings timeline chart. - - Args: - attendance_type: Optional filter for attendance type (e.g., 'onsite'). - - Returns: - Tuple of (sorted_meetings, datasets) for Chart.js. - """ - cache_key = f'stats:get_affiliation_data_for_meetings:{attendance_type}' - sorted_meetings, datasets = cache.get(cache_key, (None, None)) - if (sorted_meetings, datasets) == (None, None): - top_n = 20 # could be a parameter, but would need to adjust cache handling - - # Get registration status details - if attendance_type: - registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) - else: - registrations = Registration.objects.all() - registrations = registrations.values('affiliation', 'meeting__number') - - # Count per canonicalized affiliation - organization = dict() - meetings_set = set() - org_totals = defaultdict(int) - data_map = defaultdict(dict) # {org: {meeting: count}} - - for reg in registrations: - meeting = reg['meeting__number'] - meetings_set.add(meeting) - affiliation = canonicalize_affiliation(reg['affiliation']) or "Unspecified" - organization[affiliation] = organization.get(affiliation, 0) + 1 - org_totals[affiliation] = org_totals.get(affiliation, 0) + 1 - data_map[affiliation][meeting] = data_map[affiliation].get(meeting, 0) + 1 - - # ── Step 2: Sort meetings numerically rather than alphabetically ── - sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) - - # ── Step 3: Get top N countries ── - top_orgs = sorted( - org_totals.keys(), - key=lambda c: org_totals[c], - reverse=True - )[:top_n] - non_top_orgs = org_totals.keys() - top_orgs - other_totals = defaultdict(int) - for m in sorted_meetings: - other_totals[m] = 0 - for c in non_top_orgs: - other_totals[m] += int(data_map[c].get(m, 0)) - - # ── Step 4: Build Chart.js datasets ── - - datasets = [] - for idx, org in enumerate(top_orgs): - color = colors[idx % len(colors)] - datasets.append({ - 'label': org, - 'data': [data_map[org].get(m, 0) for m in sorted_meetings], - 'borderColor': color, - 'fill': False, - 'tension': 0.3, - 'pointColor': color, - 'pointBackgroundColor': color, - 'pointRadius': 4, - 'pointHoverRadius': 6, - 'borderWidth': 2, - }) - - # -- Step 4.bis handle the other -- - datasets.append({ - 'label': 'Other', - 'data': [other_totals.get(m, 0) for m in sorted_meetings], - 'borderColor': 'black', - 'fill': False, - 'tension': 0.3, - 'pointColor': 'black', - 'pointBackgroundColor': 'black', - 'pointRadius': 4, - 'pointHoverRadius': 6, - 'borderWidth': 2, - }) - cache.set( - cache_key, - (sorted_meetings, datasets), - settings.STATS_TIMELINE_CACHE_TIMEOUT, - ) - - return sorted_meetings, datasets - -def get_country_data_for_meetings(attendance_type=None): - """Get country participation data for meetings timeline chart. - - Args: - attendance_type: Optional filter for attendance type (e.g., 'onsite'). - - Returns: - Tuple of (sorted_meetings, datasets) for Chart.js. - """ - cache_key = f'stats:get_country_data_for_meetings:{attendance_type}' - sorted_meetings, datasets = cache.get(cache_key, (None, None)) - if (sorted_meetings, datasets) == (None, None): - top_n = 10 # could be a parameter, but would need to adjust cache handling - # Get registration status counts, aggregated by country_code - if attendance_type: - registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) - else: - registrations = Registration.objects.all() - queryset = ( - registrations - .values( - 'meeting__number', # e.g. "118", "119", "120" - 'country_code' # country code of the participant - ) - .annotate(participant_count=Count('id')) - .order_by('meeting__number') # chronological order - ) - - # ── Step 1: Collect all meetings and country totals ── - meetings_set = set() - country_totals = defaultdict(int) - data_map = defaultdict(dict) # {country: {meeting: count}} - - for row in queryset: - meeting = row['meeting__number'] - country = row['country_code'] - count = row['participant_count'] - - meetings_set.add(meeting) - country_totals[country] += count - data_map[country][meeting] = count - - # ── Step 2: Sort meetings numerically rather than alphabetically ── - sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) - - # ── Step 3: Get top N countries ── - top_countries = sorted( - country_totals.keys(), - key=lambda c: country_totals[c], - reverse=True - )[:top_n] - - # -- Step 3.bis do the 'other' category -- - non_top_countries = country_totals.keys() - top_countries - other_totals = defaultdict(int) - for m in sorted_meetings: - other_totals[m] = 0 - for c in non_top_countries: - other_totals[m] += int(data_map[c].get(m, 0)) - - # ── Step 4: Build Chart.js datasets ── - - datasets = [] - for idx, country in enumerate(top_countries): - color = colors[idx % len(colors)] - datasets.append({ - 'label': country, - 'data': [data_map[country].get(m, 0) for m in sorted_meetings], - 'borderColor': color, - 'fill': False, - 'tension': 0.3, - 'pointColor': color, - 'pointBackgroundColor': color, - 'pointRadius': 4, - 'pointHoverRadius': 6, - 'borderWidth': 2, - }) - - # -- Step 4.bis handle the other -- - datasets.append({ - 'label': 'Other', - 'data': [other_totals.get(m, 0) for m in sorted_meetings], - 'borderColor': 'black', - 'fill': False, - 'tension': 0.3, - 'pointColor': 'black', - 'pointBackgroundColor': 'black', - 'pointRadius': 4, - 'pointHoverRadius': 6, - 'borderWidth': 2, - }) - cache.set( - cache_key, - (sorted_meetings, datasets), - settings.STATS_TIMELINE_CACHE_TIMEOUT, - ) - - return sorted_meetings, datasets - -def get_data_for_meetings(): - """Get total participation data by attendance type for meetings timeline chart. - - Returns: - Tuple of (sorted_meetings, datasets) for Chart.js. - """ - cache_key = "stats:get_data_for_meetings" - sorted_meetings, datasets = cache.get(cache_key, (None, None)) - if (sorted_meetings, datasets) == (None, None): - # Get registration status counts, aggregated by ticket types - registrations = Registration.objects.filter(tickets__attendance_type__in=['onsite', 'remote']) - queryset = ( - registrations - .values( - 'meeting__number', # e.g. "118", "119", "120" - 'tickets__attendance_type' - ) - .annotate(participant_count=Count('id')) - .order_by('meeting__number') # chronological order - ) - - # ── Step 1: Collect all meetings and tickets totals ── - meetings_set = set() - tickets_totals = defaultdict(int) - data_map = defaultdict(dict) # {ticket: {meeting: count}} - - for row in queryset: - meeting = row['meeting__number'] - ticket = row['tickets__attendance_type'] - count = row['participant_count'] - - meetings_set.add(meeting) - tickets_totals[ticket] += count - data_map[ticket][meeting] = count - - # ── Step 2: Sort meetings numerically rather than alphabetically ── - sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) - ticket_types = tickets_totals.keys() - - # ── Step 4: Build Chart.js datasets ── - # Color palette for lines - colors = [ '#FF6384', '#36A2EB'] - - datasets = [] - for idx, ticket_type in enumerate(ticket_types): - color = colors[idx % len(colors)] - datasets.append({ - 'label': ticket_type, - 'data': [data_map[ticket_type].get(m, 0) for m in sorted_meetings], - 'borderColor': color, - 'backgroundColor': color + '99', # 60% opacity fill - 'fill': True, - 'tension': 0.0, - 'pointColor': color, - 'pointBackgroundColor': color, - 'pointRadius': 4, - 'pointHoverRadius': 6, - 'borderWidth': 2, - }) - cache.set( - cache_key, - (sorted_meetings, datasets), - settings.STATS_TIMELINE_CACHE_TIMEOUT, - ) - return sorted_meetings, datasets - -def meetings_timeline(request, stats_type='country'): - """Render the meetings timeline page with participation statistics over time. - - Args: - request: The HTTP request object. - stats_type: Type of statistics ('country' or 'total'). - top_n: Number of top items to show (for country stats). - - Returns: - Rendered response for the meetings timeline template. - """ - if stats_type == 'total': - total_labels, total_data_sets = get_data_for_meetings() - in_person_labels = ([], []) - in_person_data_sets = ([], []) - top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" - elif stats_type == 'affiliation': - total_labels, total_data_sets = get_affiliation_data_for_meetings() - in_person_labels, in_person_data_sets = get_affiliation_data_for_meetings(attendance_type='onsite') - top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" - elif stats_type == 'country': - total_labels, total_data_sets = get_country_data_for_meetings() - in_person_labels, in_person_data_sets = get_country_data_for_meetings(attendance_type='onsite') - top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" - else: - return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) - - total_chart_data = { - 'labels': total_labels, - 'datasets': total_data_sets, - } - - # On per country/affiliation have a separate graph for inperson - if stats_type == 'total': - in_person_chart_data = None - else: - in_person_chart_data = { - 'labels': in_person_labels, - 'datasets': in_person_data_sets, - } - - # Prepare the list of choice buttons for the template - possible_stats_types = [ - ("affiliation", "Per affiliation", urlreverse(meetings_timeline, kwargs={'stats_type': 'affiliation'})), - ("country", "Per country", urlreverse(meetings_timeline, kwargs={'stats_type': 'country'})), - ("total", "Total", urlreverse(meetings_timeline, kwargs={'stats_type': 'total'})), - ] - - current_meeting = get_current_ietf_meeting_num() - if stats_type == 'total': - possible_stats_type = 'country' - else: - possible_stats_type = stats_type - - possible_meeting_numbers = [(int(current_meeting)-1, urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting)-1, 'stats_type': possible_stats_type})), - (int(current_meeting), urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting), 'stats_type': possible_stats_type})), - (int(current_meeting)+1, urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting)+1, 'stats_type': possible_stats_type}))] - - return render(request, "stats/meetings_timeline.html", { - "top_n": top_n, - "possible_stats_types": possible_stats_types, - "possible_meeting_numbers": possible_meeting_numbers, - "stats_type": stats_type, - "total_chart_data": total_chart_data, - "in_person_chart_data": in_person_chart_data, - }) - -def get_affiliation_data_for_meeting(meeting_number, minimum_required, attendance_type=None): - """Get affiliation participation data for a specific meeting. - - Args: - meeting_number: The meeting number. - minimum_required: Minimum count to include in main data (others go to 'Other'). - attendance_type: Optional filter for attendance type. - - Returns: - Tuple of (labels, data, total) for chart display. - """ - # Get registration status details - registrations = Registration.objects.filter(meeting__number=meeting_number) - if attendance_type: - registrations = registrations.filter(tickets__attendance_type=attendance_type) - registrations = registrations.values('affiliation') - - # Count per canonicalized affiliation - organization = dict() - for reg in registrations: - affiliation = canonicalize_affiliation(reg['affiliation']) or "Unspecified" - organization[affiliation] = organization.get(affiliation, 0) + 1 - - # Sort to have the largest count first (nicer in pie chart) - sorted_orgs = sorted(organization.items(), key=lambda t: t[1], reverse=True) - labels = [] - data = [] - others_count = 0 - total = 0 - for org, count in sorted_orgs: - total += count - if count > minimum_required: - labels.append(org) - data.append(count) - else: - others_count += count - - if others_count > 0: - labels.append('Other') - data.append(others_count) - - return labels, data, total - -def get_data_for_meeting(meeting_number, minimum_required, attendance_type=None): - """Get country participation data for a specific meeting. - - Args: - meeting_number: The meeting number. - minimum_required: Minimum count to include in main data (others go to 'Other'). - attendance_type: Optional filter for attendance type. - - Returns: - Tuple of (labels, data, total) for chart display. - """ - # Get registration status counts, aggregated by country_code - registration_counts = Registration.objects.filter(meeting__number=meeting_number) - if attendance_type: - registration_counts = registration_counts.filter(tickets__attendance_type=attendance_type) - registration_counts = registration_counts.values('country_code').annotate(count=Count('country_code')).order_by('-count') - - labels = [] - data = [] - others_count = 0 - total = 0 - for item in registration_counts: - total += item['count'] - if item['count'] > minimum_required: - labels.append(item['country_code']) - data.append(item['count']) - else: - others_count += item['count'] - - if others_count > 0: - labels.append('Other') - data.append(others_count) - - return labels, data, total - -def meeting_stats(request, meeting_number=None, stats_type='country'): - """Render statistics for a specific meeting. - - Args: - request: The HTTP request object. - meeting_number: The meeting number (defaults to current). - stats_type: Type of statistics ('country' or 'affiliation'). - - Returns: - Rendered response for the meeting stats template. - """ - current_meeting_number = get_current_ietf_meeting_num() - if meeting_number is None: - meeting_number = current_meeting_number - this_meeting = get_object_or_404( - Meeting.objects.filter(type_id="ietf"), number=meeting_number - ) - - if stats_type == 'affiliation': - minimum_required = 4 - total_labels, total_data, total_total = get_affiliation_data_for_meeting(meeting_number, minimum_required) - in_person_labels, in_person_data, in_person_total = get_affiliation_data_for_meeting(meeting_number, minimum_required, attendance_type='onsite') - elif stats_type == 'country': - minimum_required = 10 - total_labels, total_data, total_total = get_data_for_meeting(meeting_number, minimum_required) - in_person_labels, in_person_data, in_person_total = get_data_for_meeting(meeting_number, minimum_required, attendance_type='onsite') - else: - return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) - - total_chart_data = { - 'labels': total_labels, - 'datasets': [{ - 'label': 'Total Registrations by ' + stats_type, - 'data': total_data, - 'borderColor': '#ffffff', - 'borderWidth': 2, - }] - } - in_person_chart_data = { - 'labels': in_person_labels, - 'datasets': [{ - 'label': 'In Person Registrations by ' + stats_type, - 'data': in_person_data, - 'borderColor': '#ffffff', - 'borderWidth': 2, - }] - } - - # Prepare the list of choice buttons for the template - possible_stats_types = [ - ("affiliation", "Per affiliation", urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': 'affiliation'})), - ("country", "Per country", urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': 'country'})), - ] - - # Prepare the list of meeting number buttons for the template - possible_meeting_numbers = [('All', urlreverse(meetings_timeline, kwargs={'stats_type': stats_type}))] - if int(meeting_number) > 72: # No registration data before IETF-72 - possible_meeting_numbers.append((int(meeting_number)-1, urlreverse(meeting_stats, kwargs={'meeting_number': int(meeting_number)-1, 'stats_type': stats_type}))) - possible_meeting_numbers.append((meeting_number, urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': stats_type}))) - if int(meeting_number) <= int(current_meeting_number): # Allow current meeting +1 - possible_meeting_numbers.append((int(meeting_number)+1, urlreverse(meeting_stats, kwargs={'meeting_number': int(meeting_number)+1, 'stats_type': stats_type}))) - - return render(request, "stats/meeting_stats.html", { - "meeting_number": meeting_number, - "meeting_date": this_meeting.date, - "meeting_country": this_meeting.country, - "meeting_city": this_meeting.city, - "possible_stats_types": possible_stats_types, - "possible_meeting_numbers": possible_meeting_numbers, - "stats_type": stats_type, - "minimum_required": minimum_required, - "total_chart_data": total_chart_data, - "total_total": total_total, - "in_person_chart_data": in_person_chart_data, - "in_person_total": in_person_total - }) - - -@login_required -def review_stats(request, stats_type=None, acronym=None): - """Render review statistics page with tables and charts for review assignments. - - Shows completion status, results, assignment states, and time series data. - Supports both team-level and reviewer-level views with filtering options. - - Args: - request: The HTTP request object. - stats_type: Type of statistics ('completion', 'results', 'states', 'time'). - acronym: Team acronym for reviewer-level view (None for team view). - - Returns: - Rendered response for the review stats template. - """ - # This view is a bit complex because we want to show a bunch of - # tables with various filtering options, and both a team overview - # and a reviewers-within-team overview - and a time series chart. - # And in order to make the UI quick to navigate, we're not using - # one big form but instead presenting a bunch of immediate - # actions, with a URL scheme where the most common options (level - # and statistics type) are incorporated directly into the URL to - # be a bit nicer. - - def build_review_stats_url(stats_type_override=Ellipsis, acronym_override=Ellipsis, get_overrides=None): - if get_overrides is None: - get_overrides = {} - kwargs = { - "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override, - } - acr = acronym if acronym_override is Ellipsis else acronym_override - if acr: - kwargs["acronym"] = acr - - return urlreverse(review_stats, kwargs=kwargs) + generate_query_string(request.GET, get_overrides) - - # which overview - team or reviewer - if acronym: - level = "reviewer" - else: - level = "team" - - # statistics type - one of the tables or the chart - possible_stats_types = [ - ("completion", "Completion status"), - ("results", "Review results"), - ("states", "Assignment states"), - ] - - if level == "team": - possible_stats_types.append(("time", "Changes over time")) - - possible_stats_types = add_url_to_choices(possible_stats_types, - lambda slug: build_review_stats_url(stats_type_override=slug)) - - if not stats_type: - return HttpResponseRedirect(build_review_stats_url(stats_type_override=possible_stats_types[0][0])) - - # what to count - possible_count_choices = add_url_to_choices([ - ("", "Review requests"), - ("pages", "Reviewed pages"), - ], lambda slug: build_review_stats_url(get_overrides={ "count": slug })) - - count = get_choice(request, "count", possible_count_choices) or "" - - # time range - def parse_date(s): - if not s: - return None - try: - return datetime.datetime.strptime(s.strip(), "%Y-%m-%d").date() - except ValueError: - return None - - today = date_today(DEADLINE_TZINFO) - from_date = parse_date(request.GET.get("from")) or today - dateutil.relativedelta.relativedelta(years=1) - to_date = parse_date(request.GET.get("to")) or today - - from_time = datetime.datetime.combine(from_date, datetime.time.min, tzinfo=DEADLINE_TZINFO) - to_time = datetime.datetime.combine(to_date, datetime.time.max, tzinfo=DEADLINE_TZINFO) - - # teams/reviewers - teams = list(Group.objects.exclude(reviewrequest=None).distinct().order_by("name")) - - reviewer_filter_args = {} - - # - interlude: access control - if has_role(request.user, ["Secretariat", "Area Director"]): - pass - else: - secr_access = set() - reviewer_only_access = set() - - for r in Role.objects.filter(person__user=request.user, name__in=["secr", "reviewer"], group__in=teams).distinct(): - if r.name_id == "secr": - secr_access.add(r.group_id) - reviewer_only_access.discard(r.group_id) - elif r.name_id == "reviewer": - if not r.group_id in secr_access: - reviewer_only_access.add(r.group_id) - - if not secr_access and not reviewer_only_access: - permission_denied(request, "You do not have the necessary permissions to view this page") - - teams = [t for t in teams if t.pk in secr_access or t.pk in reviewer_only_access] - - for t in reviewer_only_access: - reviewer_filter_args[t] = { "user": request.user } - - reviewers_for_team = None - - if level == "team": - for t in teams: - t.reviewer_stats_url = build_review_stats_url(acronym_override=t.acronym) - - query_teams = teams - query_reviewers = None - - group_by_objs = { t.pk: t for t in query_teams } - group_by_index = ReviewAssignmentData._fields.index("team") - - elif level == "reviewer": - for t in teams: - if t.acronym == acronym: - reviewers_for_team = t - break - else: - return HttpResponseRedirect(urlreverse(review_stats)) - - query_reviewers = list(Person.objects.filter( - email__reviewassignment__review_request__time__gte=from_time, - email__reviewassignment__review_request__time__lte=to_time, - email__reviewassignment__review_request__team=reviewers_for_team, - **reviewer_filter_args.get(t.pk, {}) - ).distinct()) - query_reviewers.sort(key=lambda p: p.last_name()) - - query_teams = [t] - - group_by_objs = { r.pk: r for r in query_reviewers } - group_by_index = ReviewAssignmentData._fields.index("reviewer") - - # now filter and aggregate the data - possible_teams = possible_completion_types = possible_results = possible_states = None - selected_teams = selected_completion_type = selected_result = selected_state = None - - if stats_type == "time": - possible_teams = [(t.acronym, t.acronym) for t in teams] - selected_teams = get_choice(request, "team", possible_teams, multiple=True) - - def add_if_exists_else_subtract(element, l): - if element in l: - return [x for x in l if x != element] - else: - return l + [element] - - possible_teams = add_url_to_choices( - possible_teams, - lambda slug: build_review_stats_url(get_overrides={ - "team": add_if_exists_else_subtract(slug, selected_teams) - }) - ) - query_teams = [t for t in query_teams if t.acronym in selected_teams] - - extracted_data = extract_review_assignment_data(query_teams, query_reviewers, from_time, to_time) - - req_time_index = ReviewAssignmentData._fields.index("req_time") - - def time_key_fn(t): - d = t[req_time_index].date() - #d -= datetime.timedelta(days=d.weekday()) # weekly - # NOTE: Earlier releases had an off-by-one error here - some stat counts may move a month. - d -= datetime.timedelta(days=d.day-1) # monthly - return d - - found_results = set() - found_states = set() - aggrs = [] - for d, request_data_items in itertools.groupby(extracted_data, key=time_key_fn): - raw_aggr = aggregate_raw_period_review_assignment_stats(request_data_items, count=count) - aggr = sum_period_review_assignment_stats(raw_aggr) - - aggrs.append((d, aggr)) - - for slug in aggr["result"]: - found_results.add(slug) - for slug in aggr["state"]: - found_states.add(slug) - - results = ReviewResultName.objects.filter(slug__in=found_results) - states = ReviewAssignmentStateName.objects.filter(slug__in=found_states) - - # choice - - possible_completion_types = add_url_to_choices([ - ("completed_in_time_or_late", "Completed (in time or late)"), - ("not_completed", "Not completed"), - ("average_assignment_to_closure_days", "Avg. compl. days"), - ], lambda slug: build_review_stats_url(get_overrides={ "completion": slug, "result": None, "state": None })) - - selected_completion_type = get_choice(request, "completion", possible_completion_types) - - possible_results = add_url_to_choices( - [(r.slug, r.name) for r in results], - lambda slug: build_review_stats_url(get_overrides={ "completion": None, "result": slug, "state": None }) - ) - - selected_result = get_choice(request, "result", possible_results) - - possible_states = add_url_to_choices( - [(s.slug, s.name) for s in states], - lambda slug: build_review_stats_url(get_overrides={ "completion": None, "result": None, "state": slug }) - ) - - selected_state = get_choice(request, "state", possible_states) - - if not selected_completion_type and not selected_result and not selected_state: - selected_completion_type = "completed_in_time_or_late" - - standard_color = '#3d22b3' - if selected_completion_type == 'completed_in_time_or_late': - graph_data = [ - {'label': 'in time', 'color': standard_color, 'data': []}, - {'label': 'late', 'color': '#b42222', 'data': []} - ] - else: - graph_data = [{'color': standard_color, 'data': []}] - if selected_completion_type == "completed_combined": - pass - else: - for d, aggr in aggrs: - v1 = 0 - v2 = None - js_timestamp = calendar.timegm(d.timetuple()) * 1000 - if selected_completion_type == 'completed_in_time_or_late': - v1 = aggr['completed_in_time'] - v2 = aggr['completed_late'] - elif selected_completion_type is not None: - v1 = aggr[selected_completion_type] - elif selected_result is not None: - v1 = aggr["result"][selected_result] - elif selected_state is not None: - v1 = aggr["state"][selected_state] - - graph_data[0]['data'].append((js_timestamp, v1)) - if v2 is not None: - graph_data[1]['data'].append((js_timestamp, v2)) - data = json.dumps(graph_data) - - else: # tabular data - extracted_data = extract_review_assignment_data(query_teams, query_reviewers, from_time, to_time, ordering=[level]) - - data = [] - - found_results = set() - found_states = set() - raw_aggrs = [] - for group_pk, request_data_items in itertools.groupby(extracted_data, key=lambda t: t[group_by_index]): - raw_aggr = aggregate_raw_period_review_assignment_stats(request_data_items, count=count) - raw_aggrs.append(raw_aggr) - - aggr = sum_period_review_assignment_stats(raw_aggr) - - # skip zero-valued rows - if aggr["open"] == 0 and aggr["completed"] == 0 and aggr["not_completed"] == 0: - continue - - aggr["obj"] = group_by_objs.get(group_pk) - - for slug in aggr["result"]: - found_results.add(slug) - for slug in aggr["state"]: - found_states.add(slug) - - data.append(aggr) - - # add totals row - if len(raw_aggrs) > 1: - totals = sum_period_review_assignment_stats(sum_raw_review_assignment_aggregations(raw_aggrs)) - totals["obj"] = "Totals" - data.append(totals) - - results = ReviewResultName.objects.filter(slug__in=found_results) - states = ReviewAssignmentStateName.objects.filter(slug__in=found_states) - - # massage states/results breakdowns for template rendering - for aggr in data: - aggr["state_list"] = [aggr["state"].get(x.slug, 0) for x in states] - aggr["result_list"] = [aggr["result"].get(x.slug, 0) for x in results] - - - return render(request, 'stats/review_stats.html', { - "team_level_url": build_review_stats_url(acronym_override=None), - "level": level, - "reviewers_for_team": reviewers_for_team, - "teams": teams, - "data": data, - "states": states, - "results": results, - - # options - "possible_stats_types": possible_stats_types, - "stats_type": stats_type, - - "possible_count_choices": possible_count_choices, - "count": count, - - "from_date": from_date, - "to_date": to_date, - "today": today, - - # time options - "possible_teams": possible_teams, - "selected_teams": selected_teams, - "possible_completion_types": possible_completion_types, - "selected_completion_type": selected_completion_type, - "possible_results": possible_results, - "selected_result": selected_result, - "possible_states": possible_states, - "selected_state": selected_state, - }) - - @role_required("LLC Staff") def annual_report_inputs(request, year=None): if year is None and "year" in request.GET: return HttpResponseRedirect( - urlreverse("ietf.stats.views.annual_report_inputs", kwargs={"year": request.GET["year"]}) + urlreverse("ietf.stats.views.annual_report_inputs", kwargs={"year": request.GET["year"]}), ) year = int(year) if year else datetime.date.today().year - 1 @@ -987,8 +95,8 @@ def annual_report_inputs(request, year=None): draft_count = len(set( NewRevisionDocEvent.objects.filter( - doc__type_id="draft", time__year=year - ).values_list("doc__name", flat=True) + doc__type_id="draft", time__year=year, + ).values_list("doc__name", flat=True), )) return render(request, "stats/annual_report_inputs.html", { diff --git a/ietf/stats/views_authors.py b/ietf/stats/views_authors.py new file mode 100644 index 00000000000..c0a3cbfe08c --- /dev/null +++ b/ietf/stats/views_authors.py @@ -0,0 +1,421 @@ +# Copyright The IETF Trust 2016-2026, All Rights Reserved + +from collections import Counter, defaultdict + +from django.conf import settings +from django.core.cache import cache +from django.db.models import Count, DateTimeField, OuterRef, Q, Subquery +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect +from django.shortcuts import render +from django.urls import reverse as urlreverse + +from ietf.doc.models import DocEvent, DocumentAuthor, RfcAuthor +from ietf.stats.utils import ( + color_from_hash, + get_aliased_affiliations, + get_aliased_countries, + get_top_n_choices, + get_valid_top_n, +) +from ietf.utils.timezone import RPC_TZINFO + + +def _country_not_available_error(request: HttpRequest, doc_type: str) -> HttpResponse | None: + """Return the country-stat error response when the selected doc set cannot provide it.""" + if doc_type in ["all", "rfc"]: + return render( + request, + "stats/error.html", + { + "message": f"Country information not (yet) available for {doc_type} documents from the RFC Editor.", + }, + ) + return None + + +def _get_document_type_choices(view, doc_type: str, stats_type: str) -> list[tuple[str, str, str]]: + """Build the document-type navigation choices for each stats view.""" + possible_docs_types = [ + ("draft", "Drafts", urlreverse(view, kwargs={"doc_type": "draft", "stats_type": stats_type})), + ("wg-draft", "WG Drafts", urlreverse(view, kwargs={"doc_type": "wg-draft", "stats_type": stats_type})), + ] + if stats_type != "country": + possible_docs_types = [ + ("all", "All documents", urlreverse(view, kwargs={"doc_type": "all", "stats_type": stats_type})), + ] + possible_docs_types + [ + ("rfc", "RFCs", urlreverse(view, kwargs={"doc_type": "rfc", "stats_type": stats_type})), + ] + return possible_docs_types + + +def _get_stats_type_choices(view, doc_type: str, stats_type: str) -> list[tuple[str, str, str]]: + """Build the stats-type navigation choices for each stats view.""" + possible_stats_types = [ + ("affiliation", "Affiliation", urlreverse(view, kwargs={"doc_type": doc_type, "stats_type": "affiliation"})), + ] + if doc_type not in ["all", "rfc"]: + possible_stats_types.append( + ("country", "Country", urlreverse(view, kwargs={"doc_type": doc_type, "stats_type": "country"})), + ) + return possible_stats_types + + +def get_authors_total_data_for_documents(doc_type: str = "all", + group_by: str = "country", + top_n: int = 20) -> dict[str, object]: + """Build chart data for author totals. + + Args: + doc_type: Document category to filter on: all, draft, wg-draft, rfc, + group_by: Field used to group authors, e.g., "country" or "affiliation". + top_n: Maximum number of top groups to include before aggregating into Other. + + Returns: + A Chart.js-compatible data dictionary. + + """ + # Build a dynamic query set filter to get country/affiliation + # using the appropriate model based on doc_type. + # RfcAuthor for RFC + # DocumentAuthor for other documents (i.e., drafts) + + # Using distinct=True in Count to avoid double counting authors + # who may have multiple entries in the database + if doc_type in ("draft", "wg-draft"): + filters = Q(document__type_id="draft") + if doc_type == "wg-draft": + filters &= Q(document__group__type_id="wg") + queryset = ( + DocumentAuthor.objects + .filter(filters) + .values(group_by) + .annotate(author_count=Count("person", distinct=True)) + ) + group_count_set = [ + (row.get(group_by), row.get("author_count", 0)) + for row in queryset + ] + elif doc_type == "rfc": + queryset = ( + RfcAuthor.objects + .values(group_by) + .annotate(author_count=Count("person", distinct=True)) + ) + group_count_set = [ + (row.get(group_by), row.get("author_count", 0)) + for row in queryset + ] + else: + # Cannot using COUNT() and then UNION else a draft/rfc author will be double counted. + draft_queryset = ( + DocumentAuthor.objects + .filter(document__type_id="draft") + .values_list("person_id", group_by) + .distinct() + ) + rfc_queryset = ( + RfcAuthor.objects + .values_list("person_id", group_by) + .distinct() + ) + + author_groups = set(draft_queryset) | set(rfc_queryset) + group_count_set = list(Counter( + group for _, group in author_groups + ).items()) + + + if group_by == "affiliation": + alias_map = get_aliased_affiliations(group for group, _ in group_count_set) + elif group_by == "country": + alias_map = get_aliased_countries(group for group, _ in group_count_set) + else: + alias_map = {} + + group_count_dict: dict[str, int] = {} + for group, count in group_count_set: + aliased_group = alias_map.get(group, group) + aliased_group = "Unspecified" if not aliased_group else str(aliased_group) + group_count_dict[aliased_group] = group_count_dict.get(aliased_group, 0) + count + + group_count_sorted = sorted(group_count_dict.items(), + key=lambda x: x[1], reverse=True) + top_groups = group_count_sorted[:top_n] + other_count = sum(count for _, count in group_count_sorted[top_n:]) + if other_count > 0: + top_groups.append(("Other", other_count)) + + labels, data = zip(*top_groups) if top_groups else ((), ()) + labels_list = list(labels) + data_list = list(data) + chart_data: dict[str, object] = { + "labels": labels_list, + "datasets": [{ + "data": data_list, + "backgroundColor": [color_from_hash(label) + if label else "#202020" + for label in labels_list], + "borderColor": "black", + "borderWidth": 1, + }], + } + + return chart_data + +def authors_total(request: HttpRequest, doc_type: str = "all", + stats_type: str = "affiliation") -> HttpResponse: + """Render total author statistics. + + Args: + request: The incoming HTTP request. + doc_type: Document category to filter on. + stats_type: Grouping type for statistics. + + Returns: + Rendered response for the total statistics page. + + """ + top_n, error_response = get_valid_top_n(request) + + if error_response is not None: + return error_response + + if stats_type == "affiliation": + chart_data = get_authors_total_data_for_documents(doc_type, "affiliation", top_n) + elif stats_type == "country": + error_response = _country_not_available_error(request, doc_type) + if error_response is not None: + return error_response + chart_data = get_authors_total_data_for_documents(doc_type, "country", top_n) + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + + possible_docs_types = _get_document_type_choices(authors_total, doc_type, stats_type) + possible_stats_types = _get_stats_type_choices(authors_total, doc_type, stats_type) + + return render(request, "stats/documents_total.html", { + "top_n": top_n, + "top_n_choices": get_top_n_choices(), + "objects": "authors", + "possible_docs_types": possible_docs_types, + "possible_stats_types": possible_stats_types, + "timeline_url": urlreverse(authors_timeline, kwargs={"doc_type": doc_type, "stats_type": stats_type}), + "total_url": urlreverse(authors_total, kwargs={"doc_type": doc_type, "stats_type": stats_type}), + "doc_type": doc_type, + "stats_type": stats_type, + "chart_data": chart_data, + }) + + +def get_authors_timeline_data_for_documents(doc_type: str = "all", group_by: str = "country", top_n: int = 10) -> tuple[list[int], list[dict[str, object]]]: + """Build timeline datasets for author statistics. + + Args: + doc_type: Document category to filter on: all, draft, wg-draft, rfc, + group_by: Field used to group authors, e.g., "country" or "affiliation". + top_n: Maximum number of top groups to include before aggregating into Other. + + Returns: + A tuple containing the ordered years list and Chart.js datasets. + + """ + cache_key = f"stats:get_authors_timeline_data_for_documents:{doc_type}-{group_by}" + result = cache.get(cache_key, None) + if result is not None: + years_list, documents_totals, data_map = result + else: + # Fetch each document's publication event time in the same query rather + # than triggering a separate query per row via Document.pub_date(). + new_revision_time = None + published_rfc_time = None + if (doc_type in ("draft", "wg-draft")) or (doc_type == "all"): + new_revision_time = Subquery( + DocEvent.objects + .filter(doc=OuterRef("document_id"), type="new_revision") + .order_by("-time", "-id") + .values("time")[:1], + output_field=DateTimeField(), + ) + if (doc_type == "rfc") or (doc_type == "all"): + published_rfc_time = Subquery( + DocEvent.objects + .filter(doc=OuterRef("document_id"), type="published_rfc") + .order_by("-time", "-id") + .values("time")[:1], + output_field=DateTimeField(), + ) + + # Build a dynamic query set filter to get country/affiliation using the appropriate model based on doc_type. + # RfcAuthor for RFC + # DocumentAuthor for other documents (i.e., drafts) + draft_queryset = None + rfc_queryset = None + if doc_type in ("draft", "wg-draft"): + filters = Q(document__type_id="draft") + if doc_type == "wg-draft": + filters &= Q(document__group__type_id="wg") + draft_queryset = ( + DocumentAuthor.objects + .filter(filters) + .annotate(pub_datetime=new_revision_time) + ) + elif doc_type == "rfc": + rfc_queryset = ( + RfcAuthor.objects + .annotate(pub_datetime=published_rfc_time) + ) + else: + draft_queryset = ( + DocumentAuthor.objects + .filter(document__type_id="draft") + .annotate(pub_datetime=new_revision_time) + ) + rfc_queryset = ( + RfcAuthor.objects + .annotate(pub_datetime=published_rfc_time) + ) + + # ── Step 1: Collect all authors publication dates ── + years_set = set() + documents_totals = defaultdict(int) + data_map = defaultdict(dict) + year_group_list = [] + if draft_queryset is not None: + year_group_list += [ + (row.pub_datetime.astimezone(RPC_TZINFO).year, getattr(row, group_by)) + for row in draft_queryset + if row.pub_datetime is not None + ] + if rfc_queryset is not None: + year_group_list += [ + (row.pub_datetime.astimezone(RPC_TZINFO).year, getattr(row, group_by)) + for row in rfc_queryset + if row.pub_datetime is not None + ] + + if group_by == "affiliation": + alias_map = get_aliased_affiliations(group for _, group in year_group_list) + year_group_list = [(year, alias_map.get(group, group)) for year, group in year_group_list] + elif group_by == "country": + alias_map = get_aliased_countries(group for _, group in year_group_list) + year_group_list = [(year, alias_map.get(group, group)) for year, group in year_group_list] + else: + alias_map = {} + # Let us define a value when there is none... + alias_map[""] = "Unspecified" + + years_set = {year for year, _ in year_group_list} + + for year, group in year_group_list: + aliased_group = alias_map.get(group, group) + data_map[year][aliased_group] = data_map[year].get(aliased_group, 0) + 1 + documents_totals[aliased_group] += 1 + + # ── Step 2: Sort years numerically rather than alphabetically ── + years_list = sorted(years_set) + cache.set( + cache_key, + (years_list, documents_totals, data_map), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) + + # ── Step 3: Get top N and others ── must be outside of the cache + top_groups = sorted( + documents_totals.keys(), + key=lambda c: documents_totals[c], + reverse=True, + )[:top_n] + non_top_groups = documents_totals.keys() - set(top_groups) + other_totals: dict[int, int] = defaultdict(int) + other_bin_is_empty = True + for y in years_list: + for g in non_top_groups: + count = int(data_map[y].get(g, 0)) + other_totals[y] += count + if count > 0: + other_bin_is_empty = False + + # ── Step 4: Build Chart.js datasets ── + + datasets = [] + for group in top_groups: + color = color_from_hash(group) + datasets.append({ + "label": group, + "data": [data_map[year].get(group, 0) for year in years_list], + "borderColor": color, + "backgroundColor": color + "99", # 60% opacity fill + "fill": False, + "tension": 0.0, + "pointColor": color, + "pointBackgroundColor": color, + "pointRadius": 4, + "pointHoverRadius": 6, + "borderWidth": 2, + }) + + # -- Step 4.bis handle the other -- + if not other_bin_is_empty: + datasets.append({ + "label": "Other", + "data": [other_totals.get(year, 0) for year in years_list], + "borderColor": "black", + "fill": False, + "tension": 0.0, + "pointColor": "black", + "pointBackgroundColor": "black", + "pointRadius": 4, + "pointHoverRadius": 6, + "borderWidth": 2, + }) + + return years_list, datasets + + +def authors_timeline(request: HttpRequest, doc_type: str = "all", stats_type: str = "affiliation") -> HttpResponse: + """Render author timeline statistics. + + Args: + request: The incoming HTTP request. + doc_type: Document category to filter on. + stats_type: Grouping type for statistics. + + Returns: + Rendered response for the timeline statistics page. + + """ + top_n, error_response = get_valid_top_n(request) + + if error_response is not None: + return error_response + + if stats_type == "affiliation": + total_labels, total_data_sets = get_authors_timeline_data_for_documents(doc_type, "affiliation", top_n) + elif stats_type == "country": + error_response = _country_not_available_error(request, doc_type) + if error_response is not None: + return error_response + total_labels, total_data_sets = get_authors_timeline_data_for_documents(doc_type, "country", top_n) + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + + chart_data = { + "labels": total_labels, + "datasets": total_data_sets, + } + + possible_docs_types = _get_document_type_choices(authors_timeline, doc_type, stats_type) + possible_stats_types = _get_stats_type_choices(authors_timeline, doc_type, stats_type) + + return render(request, "stats/documents_timeline.html", { + "top_n": top_n, + "top_n_choices": get_top_n_choices(), + "objects": "authors", + "possible_docs_types": possible_docs_types, + "possible_stats_types": possible_stats_types, + "timeline_url": urlreverse(authors_timeline, kwargs={"doc_type": doc_type, "stats_type": stats_type}), + "total_url": urlreverse(authors_total, kwargs={"doc_type": doc_type, "stats_type": stats_type}), + "doc_type": doc_type, + "stats_type": stats_type, + "chart_data": chart_data, + }) diff --git a/ietf/stats/views_documents.py b/ietf/stats/views_documents.py new file mode 100644 index 00000000000..10a9ab4858f --- /dev/null +++ b/ietf/stats/views_documents.py @@ -0,0 +1,313 @@ +# Copyright The IETF Trust 2016-2026, All Rights Reserved + +from collections import defaultdict +from typing import Any + +from django.conf import settings +from django.core.cache import cache +from django.db.models import Count, DateTimeField, OuterRef, Subquery +from django.http import HttpResponseRedirect +from django.shortcuts import render +from django.urls import reverse as urlreverse + +from ietf.doc.models import DocEvent, Document +from ietf.stats.utils import color_from_hash, get_top_n_choices, get_valid_top_n +from ietf.utils.timezone import RPC_TZINFO + + +def _get_document_type_choices(view, doc_type: str, stats_type: str) -> list[tuple[str, str, str]]: + """Build the document-type navigation choices for a stats view.""" + return [ + ("draft", "Drafts", urlreverse(view, kwargs={"doc_type": "draft", "stats_type": stats_type})), + ("rfc", "RFCs", urlreverse(view, kwargs={"doc_type": "rfc", "stats_type": stats_type})), + ] + + +def _get_stats_type_choices(view, doc_type: str, stats_type: str) -> list[tuple[str, str, str]]: + """Build the stats-type navigation choices for a stats view.""" + possible_stats_types = [ + ("stream", "Streams", urlreverse(view, kwargs={"doc_type": doc_type, "stats_type": "stream"})), + ("wg", "Working Groups", urlreverse(view, kwargs={"doc_type": doc_type, "stats_type": "wg"})), + ] + if doc_type == "draft": + possible_stats_types.append( + ("level", "Intended Status", urlreverse(view, kwargs={"doc_type": doc_type, "stats_type": "level"})), + ) + elif doc_type == "rfc": + possible_stats_types.append( + ("level", "Category", urlreverse(view, kwargs={"doc_type": doc_type, "stats_type": "level"})), + ) + return possible_stats_types + + +def get_total_data_for_documents( + doc_type: str = "rfc", + group_by: str = "level", + top_n: int = 20, +) -> dict[str, Any]: + """Get aggregated document statistics grouped by the specified field. + + Args: + doc_type: Document type filter ('rfc', 'draft'). + group_by: Field to group by (e.g., 'stream__name', 'group__name'). + top_n: Number of top groups to display. + + Returns: + Chart.js compatible data dictionary with labels and datasets. + + """ + queryset = ( + Document.objects + .filter(type_id=doc_type) + .values(group_by) + .annotate(document_count=Count("id", distinct=True)) + .order_by("-document_count") + ) + + # Convert queryset to dictionary, aggregating by group + group_count_dict: dict[str, int] = {} + for group, count in queryset.values_list(group_by, "document_count"): + if not group: + group = "Unspecified" + group_count_dict[group] = group_count_dict.get(group, 0) + count + + sorted_groups = sorted(group_count_dict.items(), key=lambda x: x[1], reverse=True) + top_groups = sorted_groups[:top_n] + other_count = sum(count for _, count in sorted_groups[top_n:]) + if other_count > 0: + top_groups.append(("Other", other_count)) + + labels: tuple[str, ...] = tuple(label for label, _ in top_groups) + data: tuple[int, ...] = tuple(count for _, count in top_groups) + chart_data: dict[str, Any] = { + "labels": labels, + "datasets": [{ + "data": data, + "backgroundColor": [color_from_hash(label) + if label else "#202020" + for label in labels], + "borderColor": "black", + "borderWidth": 1, + }], + } + return chart_data + +def documents_total(request: Any, doc_type: str = "rfc", stats_type: str = "level") -> Any: + """Render document statistics page with pie chart aggregations. + + Args: + request: The HTTP request object. + doc_type: Type of documents to display. + stats_type: Field to aggregate by. + + Returns: + Rendered response for the documents_total template. + + """ + top_n, error_response = get_valid_top_n(request) + + if error_response is not None: + return error_response + + if stats_type == "stream": + chart_data = get_total_data_for_documents(doc_type, "stream__name", top_n) + elif stats_type == "level" and doc_type == "draft": + chart_data = get_total_data_for_documents(doc_type, "intended_std_level__name", top_n) + elif stats_type == "level" and doc_type == "rfc": + chart_data = get_total_data_for_documents(doc_type, "std_level__name", top_n) + elif stats_type == "wg": + chart_data = get_total_data_for_documents(doc_type, "group__name", top_n) + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + + possible_docs_types = _get_document_type_choices(documents_total, doc_type, stats_type) + possible_stats_types = _get_stats_type_choices(documents_total, doc_type, stats_type) + + return render(request, "stats/documents_total.html", { + "top_n": top_n, + "top_n_choices": get_top_n_choices(), + "objects": "documents", + "possible_docs_types": possible_docs_types, + "possible_stats_types": possible_stats_types, + "timeline_url": urlreverse(documents_timeline, kwargs={"doc_type": doc_type, "stats_type": stats_type}), + "total_url": urlreverse(documents_total, kwargs={"doc_type": doc_type, "stats_type": stats_type}), + "doc_type": doc_type, + "stats_type": stats_type, + "chart_data": chart_data, + }) + +def get_timeline_data_for_documents( + doc_type: str = "rfc", + group_by: str = "stream", + top_n: int = 10, +) -> tuple[list[int], list[dict[str, Any]]]: + """Get timeline data for documents grouped by field over years. + + Args: + doc_type: Document type filter ('rfc', 'draft'). + group_by: Field to group by (e.g., 'stream', 'group', 'intended_std_level'). + top_n: Number of top groups to display. + + Returns: + Tuple of (sorted_years, datasets) for Chart.js timeline chart. + + """ + + # Initialize variables with proper types + years_set: list[int] + documents_totals: dict[str, int] + data_map: dict[int, dict[str, int]] + + cache_key = f"stats:get_timeline_data_for_documents:{doc_type}-{group_by}" + result = cache.get(cache_key, None) + if result is not None: + years_set, documents_totals, data_map = result + else: + # Fetch each document's publication event time in the same query rather + # than triggering a separate query per row via Document.pub_date(). + event_type = "published_rfc" if doc_type == "rfc" else "new_revision" + pub_datetime_subquery = Subquery( + DocEvent.objects + .filter(doc=OuterRef("pk"), type=event_type) + .order_by("-time", "-id") + .values("time")[:1], + output_field=DateTimeField(), + ) + queryset = ( + Document.objects + .filter(type_id=doc_type) + .select_related(group_by) + .annotate(pub_datetime=pub_datetime_subquery) + ) + + # ── Step 1: Collect all years and document totals ── + years_set_temp: set[int] = set() + documents_totals = defaultdict(int) + data_map = defaultdict(dict) # {year: {group: count}} + + for row in queryset: + if row.pub_datetime is None: + continue + year = row.pub_datetime.astimezone(RPC_TZINFO).year + if group_by == "stream": + group = row.stream.name if row.stream else "Unspecified" + elif group_by == "group": + group = row.group.name if row.group else "Unspecified" + elif group_by == "intended_std_level": + group = row.intended_std_level.name if row.intended_std_level else "Unspecified" + elif group_by == "std_level": + group = row.std_level.name if row.std_level else "Unspecified" + else: + group = getattr(row, group_by, None) + if not group: + group = "Unspecified" + years_set_temp.add(year) + documents_totals[group] += 1 + data_map[year][group] = data_map[year].get(group, 0) + 1 + + # ── Step 2: Sort years numerically ── + years_set = sorted(years_set_temp) + cache.set( + cache_key, + (years_set, documents_totals, data_map), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) + + top_groups = sorted( + documents_totals.keys(), + key=lambda c: documents_totals[c], + reverse=True, + )[:top_n] + non_top_groups = set(documents_totals.keys()) - set(top_groups) + other_totals: dict[int, int] = defaultdict(int) + other_bin_is_empty = True + for y in years_set: + for g in non_top_groups: + count = int(data_map[y].get(g, 0)) + other_totals[y] += count + if count > 0: + other_bin_is_empty = False + + # ── Step 3: Build Chart.js datasets ── + datasets: list[dict[str, Any]] = [] + for group in top_groups: + color = color_from_hash(group) + datasets.append({ + "label": group, + "data": [data_map[year].get(group, 0) for year in years_set], + "borderColor": color, + "backgroundColor": color + "99", # 60% opacity fill + "fill": False, + "tension": 0.0, + "pointColor": color, + "pointBackgroundColor": color, + "pointRadius": 4, + "pointHoverRadius": 6, + "borderWidth": 2, + }) + + if not other_bin_is_empty: + datasets.append({ + "label": "Other", + "data": [other_totals.get(year, 0) for year in years_set], + "borderColor": "black", + "fill": False, + "tension": 0.0, + "pointColor": "black", + "pointBackgroundColor": "black", + "pointRadius": 4, + "pointHoverRadius": 6, + "borderWidth": 2, + }) + return years_set, datasets + +def documents_timeline(request: Any, doc_type: str = "rfc", stats_type: str = "level") -> Any: + """Render the documents timeline page with document statistics over time. + + Args: + request: The HTTP request object. + doc_type: Type of documents to display. + stats_type: Field to aggregate by. + + Returns: + Rendered response for the documents timeline template. + + """ + top_n, error_response = get_valid_top_n(request) + + if error_response is not None: + return error_response + + if stats_type == "stream": + total_labels, total_data_sets = get_timeline_data_for_documents(doc_type, "stream", top_n) + elif stats_type == "level" and doc_type == "draft": + total_labels, total_data_sets = get_timeline_data_for_documents(doc_type, "intended_std_level", top_n) + elif stats_type == "level" and doc_type == "rfc": + total_labels, total_data_sets = get_timeline_data_for_documents(doc_type, "std_level", top_n) + elif stats_type == "wg": + total_labels, total_data_sets = get_timeline_data_for_documents(doc_type, "group", top_n) + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + + chart_data = { + "labels": total_labels, + "datasets": total_data_sets, + } + + possible_docs_types = _get_document_type_choices(documents_timeline, doc_type, stats_type) + possible_stats_types = _get_stats_type_choices(documents_timeline, doc_type, stats_type) + + return render(request, "stats/documents_timeline.html", { + "top_n": top_n, + "top_n_choices": get_top_n_choices(), + "objects": "documents", + "possible_docs_types": possible_docs_types, + "possible_stats_types": possible_stats_types, + "timeline_url": urlreverse(documents_timeline, + kwargs={"doc_type": doc_type, "stats_type": stats_type}), + "total_url": urlreverse(documents_total, + kwargs={"doc_type": doc_type, "stats_type": stats_type}), + "doc_type": doc_type, + "stats_type": stats_type, + "chart_data": chart_data, + }) diff --git a/ietf/stats/views_meetings.py b/ietf/stats/views_meetings.py new file mode 100644 index 00000000000..5faef020fef --- /dev/null +++ b/ietf/stats/views_meetings.py @@ -0,0 +1,618 @@ +# Copyright The IETF Trust 2016-2026, All Rights Reserved + +from collections import defaultdict +from typing import Any +import csv + +from django.conf import settings +from django.core.cache import cache +from django.db.models import Count, IntegerField +from django.db.models.functions import Cast +from django.http import HttpResponse, HttpResponseRedirect +from django.shortcuts import get_object_or_404, render +from django.urls import reverse as urlreverse + +from ietf.meeting.helpers import get_current_ietf_meeting_num +from ietf.meeting.models import Meeting, Registration +from ietf.stats.utils import ( + color_from_hash, + get_aliased_affiliations, + get_aliased_countries, + get_top_n_choices, + get_valid_top_n, +) + +# Constants +FIRST_MEETING_WITH_REGISTRATION_DATA = 72 + + +def get_meeting_stats_type_choices(view, stats_type: str, meeting_number: str | None = None) -> list[tuple[str, str, str]]: + """Build the stats-type navigation choices for meeting views.""" + kwargs = {"stats_type": stats_type} + if meeting_number is not None: + kwargs["meeting_number"] = meeting_number + + choices = [ + ("affiliation", "Per affiliation", urlreverse(view, kwargs={**kwargs, "stats_type": "affiliation"})), + ("country", "Per country", urlreverse(view, kwargs={**kwargs, "stats_type": "country"})), + ] + if view is meetings_timeline: + choices.append( + ("reg_type", "Registration type", urlreverse(view, kwargs={"stats_type": "reg_type"})), + ) + return choices + + +def get_meeting_number_choices(stats_type: str, displayed_meeting: str | int, current_meeting: int) -> list[tuple[str | int, str]]: + """Build the meeting navigation choices for timeline and detail pages.""" + + if isinstance(displayed_meeting, str) and displayed_meeting == "All": + middle_meeting = int(current_meeting) + else: + middle_meeting = int(displayed_meeting) + choices: list[tuple[str | int, str]] = [ + ("All", urlreverse(meetings_timeline, kwargs={"stats_type": stats_type})), + ] + meetings = Meeting.objects.filter(type_id="ietf").annotate( + number_as_int=Cast("number", IntegerField()), + ).filter( + number_as_int__range=(middle_meeting - 1, middle_meeting + 1), + ).order_by("number_as_int") + for meeting in meetings: + if meeting.number_as_int >= FIRST_MEETING_WITH_REGISTRATION_DATA: + choices.append( + (int(meeting.number), urlreverse(meeting_stats, kwargs={"meeting_number": meeting.number, "stats_type": stats_type})), + ) + return choices + + +def _build_timeline_datasets( + top_items: list[str], + data_map: dict[str, dict[str, int]], + sorted_meetings: list[str], + other_totals: dict[str, int], + include_background_color: bool = False, +) -> list[dict[str, Any]]: + """Build Chart.js datasets for timeline charts. + + Args: + top_items: List of top item labels (countries, affiliations, etc.). + data_map: Mapping of {item: {meeting: count}}. + sorted_meetings: Sorted list of meeting numbers. + other_totals: Mapping of {meeting: count} for 'Other' category. + include_background_color: Whether to include backgroundColor (for filled areas). + + Returns: + List of Chart.js dataset dictionaries. + + """ + datasets: list[dict[str, Any]] = [] + for item in top_items: + color = color_from_hash(item) + dataset = { + "label": item, + "data": [data_map[item].get(m, 0) for m in sorted_meetings], + "borderColor": color, + "fill": bool(include_background_color), + "tension": 0.0 if include_background_color else 0.3, + "pointColor": color, + "pointBackgroundColor": color, + "pointRadius": 4, + "pointHoverRadius": 6, + "borderWidth": 2, + } + if include_background_color: + dataset["backgroundColor"] = color + "99" + datasets.append(dataset) + + # Add "Other" category, only if it has any non-zero value + if any(other_totals.get(m, 0) for m in sorted_meetings): + datasets.append({ + "label": "Other", + "data": [other_totals.get(m, 0) for m in sorted_meetings], + "borderColor": "black", + "fill": bool(include_background_color), + "tension": 0.0 if include_background_color else 0.3, + "pointColor": "black", + "pointBackgroundColor": "black", + "pointRadius": 4, + "pointHoverRadius": 6, + "borderWidth": 2, + }) + if include_background_color: + datasets[-1]["backgroundColor"] = "#00000099" + + return datasets + + +def _build_pie_chart_data( + items_with_counts: list[tuple[str, int]], + top_n: int = 20, +) -> tuple[list[str], list[int], int]: + """Build pie chart data from sorted items. + + Args: + items_with_counts: List of (label, count) tuples, already sorted. + top_n: Number of top items to display. + + Returns: + Tuple of (labels, data, total). + + """ + labels: list[str] = [] + data: list[int] = [] + total = 0 + + for item, count in items_with_counts[:top_n]: + total += count + labels.append(item) + data.append(count) + + other_total = 0 + for _, count in items_with_counts[top_n:]: + other_total += count + total += count + + if other_total > 0: + labels.append("Other") + data.append(other_total) + + return labels, data, total + + +def get_affiliation_data_for_meetings(attendance_type: str | None = None, + top_n: int = 20) -> tuple[list[str], list[dict[str, Any]]]: + """Get affiliation participation data for meetings timeline chart. + + Args: + attendance_type: Optional filter for attendance type (e.g., 'onsite'). + top_n: Number of top items to return. + + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + + """ + cache_key = f"stats:get_affiliation_data_for_meetings:{attendance_type}" + sorted_meetings, sorted_orgs, org_totals, data_map = cache.get(cache_key, (None, None, None, None)) + if (sorted_meetings, sorted_orgs, org_totals, data_map) == (None, None, None, None): + + # Get registration status details + if attendance_type: + base_registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) + else: + base_registrations = Registration.objects.all() + registrations = list( + base_registrations.values("affiliation", "meeting__number") + ) + + # Prepare affiliation data, applying canonicalization and aliasing + alias_map = get_aliased_affiliations( + registration["affiliation"] for registration in registrations + ) + + # Count per canonicalized affiliation + meetings_set: set[str] = set() + org_totals = defaultdict(int) + data_map = defaultdict(dict) # {org: {meeting: count}} + + for reg in registrations: + meeting = reg["meeting__number"] + meetings_set.add(meeting) + if not reg["affiliation"] or not reg["affiliation"].strip(): + affiliation = "Unspecified" + else: + affiliation = alias_map.get(reg["affiliation"], reg["affiliation"]) + org_totals[affiliation] = org_totals.get(affiliation, 0) + 1 + data_map[affiliation][meeting] = data_map[affiliation].get(meeting, 0) + 1 + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x)) + + # ── Step 3: Get top N countries ── + sorted_orgs = sorted( + org_totals.keys(), + key=lambda c: org_totals[c], + reverse=True, + ) + cache.set( + cache_key, + (sorted_meetings, sorted_orgs, org_totals, data_map), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) + top_orgs = sorted_orgs[:top_n] + non_top_orgs = set(org_totals.keys()) - set(top_orgs) + other_totals: dict[str, int] = defaultdict(int) + for m in sorted_meetings: + other_totals[m] = 0 + for c in non_top_orgs: + other_totals[m] += int(data_map[c].get(m, 0)) + + # ── Step 4: Build Chart.js datasets ── + datasets = _build_timeline_datasets(top_orgs, data_map, sorted_meetings, other_totals) + + return sorted_meetings, datasets + +def get_country_data_for_meetings(attendance_type: str | None = None, + top_n: int = 20) -> tuple[list[str], list[dict[str, Any]]]: + """Get country participation data for meetings timeline chart. + + Args: + attendance_type: Optional filter for attendance type (e.g., 'onsite'). + top_n: Number of top items to return. + + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + + """ + cache_key = f"stats:get_country_data_for_meetings:{attendance_type}" + sorted_meetings, sorted_countries = cache.get(cache_key, (None, None)) + if (sorted_meetings, sorted_countries) == (None, None): + # Get registration status counts, aggregated by country_code + if attendance_type: + base_registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) + else: + base_registrations = Registration.objects.all() + queryset = ( + base_registrations + .values( + "meeting__number", # e.g. "118", "119", "120" + "country_code", # country code of the participant + ) + .annotate(participant_count=Count("id")) + .order_by("meeting__number") # chronological order + ) + + # Prepare country data, applying canonicalization and aliasing + # Mainly used to conver 2-letter country code into a full name + alias_map = get_aliased_countries(country_code + for country_code + in queryset.values_list("country_code", flat=True)) + + # ── Step 1: Collect all meetings and country totals ── + meetings_set: set[str] = set() + country_totals: dict[str, int] = defaultdict(int) + data_map: dict[str, dict[str, int]] = defaultdict(dict) # {country: {meeting: count}} + + for row in queryset: + meeting = row["meeting__number"] + country = alias_map.get(row["country_code"], row["country_code"]) + count = row["participant_count"] + + meetings_set.add(meeting) + country_totals[country] += count + data_map[country][meeting] = data_map[country].get(meeting, 0) + count + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x)) + + # ── Step 3: Get top N countries ── + sorted_countries = sorted( + country_totals.keys(), + key=lambda c: country_totals[c], + reverse=True, + ) + cache.set( + cache_key, + (sorted_meetings, sorted_countries), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) + + top_countries = sorted_countries[:top_n] + + # -- Step 3.bis do the 'other' category -- + non_top_countries = set(country_totals.keys()) - set(top_countries) + other_totals: dict[str, int] = defaultdict(int) + for m in sorted_meetings: + other_totals[m] = 0 + for c in non_top_countries: + other_totals[m] += int(data_map[c].get(m, 0)) + + # ── Step 4: Build Chart.js datasets ── + datasets = _build_timeline_datasets(top_countries, data_map, sorted_meetings, other_totals) + + return sorted_meetings, datasets + +def get_data_for_meetings(top_n: int = 20) -> tuple[list[str], list[dict[str, Any]]]: + """Get total participation data by attendance type for meetings timeline chart. + + Args: + top_n: Number of top items to display in the chart. + + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + + """ + cache_key = f"stats:get_data_for_meetings:{top_n}" + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + # Get registration status counts, aggregated by ticket types + base_registrations = ( + Registration.objects + .filter(tickets__attendance_type__in=["onsite", "remote"]) + ) + queryset = ( + base_registrations + .values( + "meeting__number", # e.g. "118", "119", "120" + "tickets__attendance_type", + ) + .annotate(participant_count=Count("id")) + .order_by("meeting__number") # chronological order + ) + + # ── Step 1: Collect all meetings and tickets totals ── + meetings_set: set[str] = set() + tickets_totals: dict[str, int] = defaultdict(int) + data_map: dict[str, dict[str, int]] = defaultdict(dict) # {ticket: {meeting: count}} + + for row in queryset: + meeting = row["meeting__number"] + ticket = row["tickets__attendance_type"] + count = row["participant_count"] + + meetings_set.add(meeting) + tickets_totals[ticket] += count + data_map[ticket][meeting] = count + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x)) + ticket_types = tickets_totals.keys() + + # ── Step 4: Build Chart.js datasets ── + datasets = _build_timeline_datasets(list(ticket_types), data_map, sorted_meetings, {}, include_background_color=True) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) + return sorted_meetings, datasets + +def meetings_timeline(request: Any, stats_type: str = "country") -> Any: + """Render the meetings timeline page with participation statistics over time. + + Args: + request: The HTTP request object. + stats_type: Type of statistics ('country' or 'total'). + + Returns: + Rendered response for the meetings timeline template. + + """ + top_n, error_response = get_valid_top_n(request) + + if error_response is not None: + return error_response + + if stats_type == "reg_type": + total_labels, total_data_sets = get_data_for_meetings(top_n=top_n) + in_person_labels: list[str] = [] + in_person_data_sets: list[dict[str, Any]] = [] + plural_stats_type = "registration types" + elif stats_type == "affiliation": + total_labels, total_data_sets = get_affiliation_data_for_meetings(top_n=top_n) + in_person_labels, in_person_data_sets = get_affiliation_data_for_meetings(attendance_type="onsite", top_n=top_n) + plural_stats_type = "affiliations" + elif stats_type == "country": + total_labels, total_data_sets = get_country_data_for_meetings(top_n=top_n) + in_person_labels, in_person_data_sets = get_country_data_for_meetings(attendance_type="onsite", top_n=top_n) + plural_stats_type = "countries" + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + + # Handle the download of CSV data if requested + download = request.GET.get("download") + if download in ("total", "in_person"): + if download == "total": + labels, data_sets = total_labels, total_data_sets + else: + labels, data_sets = in_person_labels, in_person_data_sets + + response = HttpResponse(content_type="text/csv") + # Let's set the filename to include the stats_type, download type, and meeting number (even if template sets it, this ensures the correct filename is used) + response["Content-Disposition"] = f'attachment; filename="{stats_type}-{download}-all.csv"' + writer = csv.writer(response, quoting=csv.QUOTE_NONNUMERIC, lineterminator="\n", dialect="excel") + writer.writerow(["IETF meeting", stats_type, "count"]) + for meeting_nr in labels: + for ds in data_sets: + count = ds["data"][labels.index(meeting_nr)] + writer.writerow([int(meeting_nr), ds["label"], count]) + return response + + # Not for download, prepare the chart data for rendering in the template + total_chart_data = { + "labels": total_labels, + "datasets": total_data_sets, + } + + # On per country/affiliation have a separate graph for inperson + if stats_type == "reg_type": + in_person_chart_data = None + else: + in_person_chart_data = { + "labels": in_person_labels, + "datasets": in_person_data_sets, + } + + possible_stats_types = get_meeting_stats_type_choices(meetings_timeline, stats_type) + + current_meeting = get_current_ietf_meeting_num() + possible_stats_type = "country" if stats_type == "reg_type" else stats_type + possible_meeting_numbers = get_meeting_number_choices(possible_stats_type, "All", current_meeting) + + return render(request, "stats/meetings_timeline.html", { + "top_n": top_n, + "top_n_choices": get_top_n_choices(), + "possible_stats_types": possible_stats_types, + "possible_meeting_numbers": possible_meeting_numbers, + "stats_type": stats_type, + "plural_stats_type": plural_stats_type, + "total_chart_data": total_chart_data, + "in_person_chart_data": in_person_chart_data, + }) + +def get_affiliation_data_for_meeting(meeting_number: str, top_n: int = 20, + attendance_type: str | None = None) -> tuple[list[str], list[int], int]: + """Get affiliation participation data for a specific meeting. + + Args: + meeting_number: The meeting number. + top_n: Number of top items to display in the chart. + attendance_type: Optional filter for attendance type. + + Returns: + Tuple of (labels, data, total) for chart.js display. + + """ + # Get registration status details + base_registrations = Registration.objects.filter(meeting__number=meeting_number) + if attendance_type: + base_registrations = base_registrations.filter(tickets__attendance_type=attendance_type) + registrations = base_registrations.values("affiliation") + + alias_map = get_aliased_affiliations(affiliation for affiliation + in registrations.values_list("affiliation", flat=True)) + + # Count per canonicalized affiliation + organization: dict[str, int] = {} + for reg in registrations: + if not reg["affiliation"] or not reg["affiliation"].strip(): + affiliation = "Unspecified" + else: + affiliation = alias_map.get(reg["affiliation"], reg["affiliation"]) + organization[affiliation] = organization.get(affiliation, 0) + 1 + + # Sort to have the largest count first (nicer in pie chart) + sorted_orgs = sorted(organization.items(), key=lambda t: t[1], reverse=True) + return _build_pie_chart_data(sorted_orgs, top_n) + +def get_country_data_for_meeting(meeting_number: str, top_n: int = 20, + attendance_type: str | None = None) -> tuple[list[str], list[int], int]: + """Get country participation data for a specific meeting. + + Args: + meeting_number: The meeting number. + top_n: Number of top items to display in the chart. + attendance_type: Optional filter for attendance type. + + Returns: + Tuple of (labels, data, total) for chart.js display. + + """ + # Get registration status counts, aggregated by country_code + base_registration_counts = Registration.objects.filter(meeting__number=meeting_number) + if attendance_type: + base_registration_counts = ( + base_registration_counts + .filter(tickets__attendance_type=attendance_type) + ) + registration_counts = ( + base_registration_counts + .values("country_code") + .annotate(count=Count("country_code")) + .order_by("-count") + ) + + alias_map = get_aliased_countries(reg for reg in registration_counts.values_list("country_code", flat=True)) + + # Convert queryset to list of (label, count) tuples + items_with_counts = [ + (alias_map.get(item["country_code"], item["country_code"]), item["count"]) + for item in registration_counts + ] + + return _build_pie_chart_data(items_with_counts, top_n) + +def meeting_stats(request: Any, meeting_number: str | None = None, stats_type: str = "country") -> Any: + """Render statistics for a specific meeting. + + Args: + request: The HTTP request object. + meeting_number: The meeting number (defaults to current). + stats_type: Type of statistics ('country' or 'affiliation'). + + Returns: + Rendered response for the meeting stats template. + + """ + current_meeting_number = get_current_ietf_meeting_num() + if meeting_number is None: + meeting_number = current_meeting_number + this_meeting = get_object_or_404( + Meeting.objects.filter(type_id="ietf"), number=meeting_number, + ) + + top_n, error_response = get_valid_top_n(request) + + if error_response is not None: + return error_response + + if stats_type == "affiliation": + total_labels, total_data, total_total = get_affiliation_data_for_meeting(meeting_number, top_n=top_n) + in_person_labels, in_person_data, in_person_total = get_affiliation_data_for_meeting(meeting_number, top_n=top_n, attendance_type="onsite") + elif stats_type == "country": + total_labels, total_data, total_total = get_country_data_for_meeting(meeting_number, top_n=top_n) + in_person_labels, in_person_data, in_person_total = get_country_data_for_meeting(meeting_number, top_n=top_n, attendance_type="onsite") + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + + # Handle the download of CSV data if requested + download = request.GET.get("download") + if download in ("total", "in_person"): + if download == "total": + labels, data = total_labels, total_data + else: + labels, data = in_person_labels, in_person_data + + response = HttpResponse(content_type="text/csv") + # Let's set the filename to include the stats_type, download type, and meeting number (even if template sets it, this ensures the correct filename is used) + response["Content-Disposition"] = f'attachment; filename="{stats_type}-{download}-{meeting_number}.csv"' + writer = csv.writer(response, quoting=csv.QUOTE_NONNUMERIC, lineterminator="\n", dialect="excel") + writer.writerow([stats_type, "count"]) + for label, count in zip(labels, data): + writer.writerow([label, count]) + return response + + # Not for download, prepare the chart data for rendering in the template + total_chart_data = { + "labels": total_labels, + "datasets": [{ + "label": f"Total Registrations by {stats_type}", + "data": total_data, + "backgroundColor": [color_from_hash(label) + if label else "#202020" + for label in total_labels], + "borderColor": "#ffffff", + "borderWidth": 2, + }], + } + in_person_chart_data = { + "labels": in_person_labels, + "datasets": [{ + "label": f"In Person Registrations by {stats_type}", + "data": in_person_data, + "backgroundColor": [color_from_hash(label) + if label else "#202020" + for label in in_person_labels], + "borderColor": "#ffffff", + "borderWidth": 2, + }], + } + + possible_stats_types = get_meeting_stats_type_choices(meeting_stats, stats_type, meeting_number) + possible_meeting_numbers = get_meeting_number_choices(stats_type, meeting_number, int(current_meeting_number)) + + return render(request, "stats/meeting_stats.html", { + "meeting_number": meeting_number, + "meeting_date": this_meeting.date, + "meeting_country": this_meeting.country, + "meeting_city": this_meeting.city, + "possible_stats_types": possible_stats_types, + "possible_meeting_numbers": possible_meeting_numbers, + "stats_type": stats_type, + "top_n": top_n, + "top_n_choices": get_top_n_choices(), + "total_chart_data": total_chart_data, + "total_total": total_total, + "in_person_chart_data": in_person_chart_data, + "in_person_total": in_person_total, + }) diff --git a/ietf/stats/views_reviews.py b/ietf/stats/views_reviews.py new file mode 100644 index 00000000000..d9bc06c0edc --- /dev/null +++ b/ietf/stats/views_reviews.py @@ -0,0 +1,417 @@ +# Copyright The IETF Trust 2016-2026, All Rights Reserved +# -*- coding: utf-8 -*- + +import datetime +import calendar +import itertools +import json +import dateutil.relativedelta + +from django.contrib.auth.decorators import login_required +from django.http import HttpResponseRedirect +from django.shortcuts import render +from django.urls import reverse as urlreverse + +import debug # pyflakes:ignore +from ietf.ietfauth.utils import has_role +from ietf.utils.response import permission_denied +from ietf.utils.timezone import date_today, DEADLINE_TZINFO +from ietf.group.models import Role, Group +from ietf.person.models import Person +from ietf.name.models import ReviewResultName, ReviewAssignmentStateName +from ietf.review.utils import (extract_review_assignment_data, + aggregate_raw_period_review_assignment_stats, + ReviewAssignmentData, + sum_period_review_assignment_stats, + sum_raw_review_assignment_aggregations, + ) + +def generate_query_string(query_dict, overrides): + """ + Returns: + A query string starting with '?' if there are parameters, empty string otherwise. + """ + query_part = "" + + if query_dict or overrides: + d = query_dict.copy() + for k, v in overrides.items(): + if type(v) in (list, tuple): + if not v: + if k in d: + del d[k] + else: + d.setlist(k, v) + else: + if v is None or v == "": + if k in d: + del d[k] + else: + d[k] = v + + if d: + query_part = "?" + d.urlencode() + + return query_part + + +def get_choice(request, get_parameter, possible_choices, multiple=False): + """Extract a choice from the request GET parameters. + + Since statistics pages use links for navigation instead of forms, + this helper selects between possible choices from the URL parameters. + + Args: + request: The HTTP request object. + get_parameter: The name of the GET parameter. + possible_choices: List of tuples (value, label). + multiple: If True, return a list of found values; otherwise return the first found or None. + + Returns: + The selected value(s) or None. + """ + values = request.GET.getlist(get_parameter) + found = [t[0] for t in possible_choices if t[0] in values] + + if multiple: + return found + else: + if found: + return found[0] + else: + return None + +def add_url_to_choices(choices, url_builder): + """Add URLs to a list of choices. + + Args: + choices: List of tuples (slug, label). + url_builder: Function that takes a slug and returns a URL. + + Returns: + List of tuples (slug, label, url). + """ + return [ (slug, label, url_builder(slug)) for slug, label in choices] + +@login_required +def review_stats(request, stats_type=None, acronym=None): + """Render review statistics page with tables and charts for review assignments. + + Shows completion status, results, assignment states, and time series data. + Supports both team-level and reviewer-level views with filtering options. + + Args: + request: The HTTP request object. + stats_type: Type of statistics ('completion', 'results', 'states', 'time'). + acronym: Team acronym for reviewer-level view (None for team view). + + Returns: + Rendered response for the review stats template. + """ + # This view is a bit complex because we want to show a bunch of + # tables with various filtering options, and both a team overview + # and a reviewers-within-team overview - and a time series chart. + # And in order to make the UI quick to navigate, we're not using + # one big form but instead presenting a bunch of immediate + # actions, with a URL scheme where the most common options (level + # and statistics type) are incorporated directly into the URL to + # be a bit nicer. + + def build_review_stats_url(stats_type_override=Ellipsis, acronym_override=Ellipsis, get_overrides=None): + if get_overrides is None: + get_overrides = {} + kwargs = { + "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override, + } + acr = acronym if acronym_override is Ellipsis else acronym_override + if acr: + kwargs["acronym"] = acr + + return urlreverse(review_stats, kwargs=kwargs) + generate_query_string(request.GET, get_overrides) + + # which overview - team or reviewer + if acronym: + level = "reviewer" + else: + level = "team" + + # statistics type - one of the tables or the chart + possible_stats_types = [ + ("completion", "Completion status"), + ("results", "Review results"), + ("states", "Assignment states"), + ] + + if level == "team": + possible_stats_types.append(("time", "Changes over time")) + + possible_stats_types = add_url_to_choices(possible_stats_types, + lambda slug: build_review_stats_url(stats_type_override=slug)) + + if not stats_type: + return HttpResponseRedirect(build_review_stats_url(stats_type_override=possible_stats_types[0][0])) + + # what to count + possible_count_choices = add_url_to_choices([ + ("", "Review requests"), + ("pages", "Reviewed pages"), + ], lambda slug: build_review_stats_url(get_overrides={ "count": slug })) + + count = get_choice(request, "count", possible_count_choices) or "" + + # time range + def parse_date(s): + if not s: + return None + try: + return datetime.datetime.strptime(s.strip(), "%Y-%m-%d").date() + except ValueError: + return None + + today = date_today(DEADLINE_TZINFO) + from_date = parse_date(request.GET.get("from")) or today - dateutil.relativedelta.relativedelta(years=1) + to_date = parse_date(request.GET.get("to")) or today + + from_time = datetime.datetime.combine(from_date, datetime.time.min, tzinfo=DEADLINE_TZINFO) + to_time = datetime.datetime.combine(to_date, datetime.time.max, tzinfo=DEADLINE_TZINFO) + + # teams/reviewers + teams = list(Group.objects.exclude(reviewrequest=None).distinct().order_by("name")) + + reviewer_filter_args = {} + + # - interlude: access control + if has_role(request.user, ["Secretariat", "Area Director"]): + pass + else: + secr_access = set() + reviewer_only_access = set() + + for r in Role.objects.filter(person__user=request.user, name__in=["secr", "reviewer"], group__in=teams).distinct(): + if r.name_id == "secr": + secr_access.add(r.group_id) + reviewer_only_access.discard(r.group_id) + elif r.name_id == "reviewer": + if r.group_id not in secr_access: + reviewer_only_access.add(r.group_id) + + if not secr_access and not reviewer_only_access: + permission_denied(request, "You do not have the necessary permissions to view this page") + + teams = [t for t in teams if t.pk in secr_access or t.pk in reviewer_only_access] + + for t in reviewer_only_access: + reviewer_filter_args[t] = { "user": request.user } + + reviewers_for_team = None + + if level == "team": + for t in teams: + t.reviewer_stats_url = build_review_stats_url(acronym_override=t.acronym) + + query_teams = teams + query_reviewers = None + + group_by_objs = { t.pk: t for t in query_teams } + group_by_index = ReviewAssignmentData._fields.index("team") + + elif level == "reviewer": + for t in teams: + if t.acronym == acronym: + reviewers_for_team = t + break + else: + return HttpResponseRedirect(urlreverse(review_stats)) + + query_reviewers = list(Person.objects.filter( + email__reviewassignment__review_request__time__gte=from_time, + email__reviewassignment__review_request__time__lte=to_time, + email__reviewassignment__review_request__team=reviewers_for_team, + **reviewer_filter_args.get(t.pk, {}) + ).distinct()) + query_reviewers.sort(key=lambda p: p.last_name()) + + query_teams = [t] + + group_by_objs = { r.pk: r for r in query_reviewers } + group_by_index = ReviewAssignmentData._fields.index("reviewer") + + # now filter and aggregate the data + possible_teams = possible_completion_types = possible_results = possible_states = None + selected_teams = selected_completion_type = selected_result = selected_state = None + + if stats_type == "time": + possible_teams = [(t.acronym, t.acronym) for t in teams] + selected_teams = get_choice(request, "team", possible_teams, multiple=True) + + def add_if_exists_else_subtract(element, l): + if element in l: + return [x for x in l if x != element] + else: + return l + [element] + + possible_teams = add_url_to_choices( + possible_teams, + lambda slug: build_review_stats_url(get_overrides={ + "team": add_if_exists_else_subtract(slug, selected_teams) + }) + ) + query_teams = [t for t in query_teams if t.acronym in selected_teams] + + extracted_data = extract_review_assignment_data(query_teams, query_reviewers, from_time, to_time) + + req_time_index = ReviewAssignmentData._fields.index("req_time") + + def time_key_fn(t): + d = t[req_time_index].date() + #d -= datetime.timedelta(days=d.weekday()) # weekly + # NOTE: Earlier releases had an off-by-one error here - some stat counts may move a month. + d -= datetime.timedelta(days=d.day-1) # monthly + return d + + found_results = set() + found_states = set() + aggrs = [] + for d, request_data_items in itertools.groupby(extracted_data, key=time_key_fn): + raw_aggr = aggregate_raw_period_review_assignment_stats(request_data_items, count=count) + aggr = sum_period_review_assignment_stats(raw_aggr) + + aggrs.append((d, aggr)) + + for slug in aggr["result"]: + found_results.add(slug) + for slug in aggr["state"]: + found_states.add(slug) + + results = ReviewResultName.objects.filter(slug__in=found_results) + states = ReviewAssignmentStateName.objects.filter(slug__in=found_states) + + # choice + + possible_completion_types = add_url_to_choices([ + ("completed_in_time_or_late", "Completed (in time or late)"), + ("not_completed", "Not completed"), + ("average_assignment_to_closure_days", "Avg. compl. days"), + ], lambda slug: build_review_stats_url(get_overrides={ "completion": slug, "result": None, "state": None })) + + selected_completion_type = get_choice(request, "completion", possible_completion_types) + + possible_results = add_url_to_choices( + [(r.slug, r.name) for r in results], + lambda slug: build_review_stats_url(get_overrides={ "completion": None, "result": slug, "state": None }) + ) + + selected_result = get_choice(request, "result", possible_results) + + possible_states = add_url_to_choices( + [(s.slug, s.name) for s in states], + lambda slug: build_review_stats_url(get_overrides={ "completion": None, "result": None, "state": slug }) + ) + + selected_state = get_choice(request, "state", possible_states) + + if not selected_completion_type and not selected_result and not selected_state: + selected_completion_type = "completed_in_time_or_late" + + standard_color = '#3d22b3' + if selected_completion_type == 'completed_in_time_or_late': + graph_data = [ + {'label': 'in time', 'color': standard_color, 'data': []}, + {'label': 'late', 'color': '#b42222', 'data': []} + ] + else: + graph_data = [{'color': standard_color, 'data': []}] + if selected_completion_type == "completed_combined": + pass + else: + for d, aggr in aggrs: + v1 = 0 + v2 = None + js_timestamp = calendar.timegm(d.timetuple()) * 1000 + if selected_completion_type == 'completed_in_time_or_late': + v1 = aggr['completed_in_time'] + v2 = aggr['completed_late'] + elif selected_completion_type is not None: + v1 = aggr[selected_completion_type] + elif selected_result is not None: + v1 = aggr["result"][selected_result] + elif selected_state is not None: + v1 = aggr["state"][selected_state] + + graph_data[0]['data'].append((js_timestamp, v1)) + if v2 is not None: + graph_data[1]['data'].append((js_timestamp, v2)) + data = json.dumps(graph_data) + + else: # tabular data + extracted_data = extract_review_assignment_data(query_teams, query_reviewers, from_time, to_time, ordering=[level]) + + data = [] + + found_results = set() + found_states = set() + raw_aggrs = [] + for group_pk, request_data_items in itertools.groupby(extracted_data, key=lambda t: t[group_by_index]): + raw_aggr = aggregate_raw_period_review_assignment_stats(request_data_items, count=count) + raw_aggrs.append(raw_aggr) + + aggr = sum_period_review_assignment_stats(raw_aggr) + + # skip zero-valued rows + if aggr["open"] == 0 and aggr["completed"] == 0 and aggr["not_completed"] == 0: + continue + + aggr["obj"] = group_by_objs.get(group_pk) + + for slug in aggr["result"]: + found_results.add(slug) + for slug in aggr["state"]: + found_states.add(slug) + + data.append(aggr) + + # add totals row + if len(raw_aggrs) > 1: + totals = sum_period_review_assignment_stats(sum_raw_review_assignment_aggregations(raw_aggrs)) + totals["obj"] = "Totals" + data.append(totals) + + results = ReviewResultName.objects.filter(slug__in=found_results) + states = ReviewAssignmentStateName.objects.filter(slug__in=found_states) + + # massage states/results breakdowns for template rendering + for aggr in data: + aggr["state_list"] = [aggr["state"].get(x.slug, 0) for x in states] + aggr["result_list"] = [aggr["result"].get(x.slug, 0) for x in results] + + + return render(request, 'stats/review_stats.html', { + "team_level_url": build_review_stats_url(acronym_override=None), + "level": level, + "reviewers_for_team": reviewers_for_team, + "teams": teams, + "data": data, + "states": states, + "results": results, + + # options + "possible_stats_types": possible_stats_types, + "stats_type": stats_type, + + "possible_count_choices": possible_count_choices, + "count": count, + + "from_date": from_date, + "to_date": to_date, + "today": today, + + # time options + "possible_teams": possible_teams, + "selected_teams": selected_teams, + "possible_completion_types": possible_completion_types, + "selected_completion_type": selected_completion_type, + "possible_results": possible_results, + "selected_result": selected_result, + "possible_states": possible_states, + "selected_state": selected_state, + }) diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html index 905c337850e..b70788fbbdc 100644 --- a/ietf/templates/base/menu.html +++ b/ietf/templates/base/menu.html @@ -441,31 +441,50 @@ diff --git a/ietf/templates/group/review_requests.html b/ietf/templates/group/review_requests.html index 61b93de2b66..4e5851a2e9c 100644 --- a/ietf/templates/group/review_requests.html +++ b/ietf/templates/group/review_requests.html @@ -11,7 +11,7 @@ {% origin %} {% if can_access_stats %}
- +
diff --git a/ietf/templates/group/reviewer_overview.html b/ietf/templates/group/reviewer_overview.html index 75bd15f1d30..a603bd92035 100644 --- a/ietf/templates/group/reviewer_overview.html +++ b/ietf/templates/group/reviewer_overview.html @@ -11,7 +11,7 @@ {% origin %} {% if can_access_stats %}
- diff --git a/ietf/templates/stats/documents_timeline.html b/ietf/templates/stats/documents_timeline.html new file mode 100644 index 00000000000..5f60e881efd --- /dev/null +++ b/ietf/templates/stats/documents_timeline.html @@ -0,0 +1,78 @@ +{% extends "base.html" %} +{% load origin %} +{% load ietf_filters static django_bootstrap5 %} +{% block js %} + {{ chart_data|json_script:"chart_data" }} + {{ objects|json_script:"objects" }} + +{% endblock %} +{% block content %} + {% origin %} +

+ {% block title %} + Statistics for IETF {{ objects|title }} + {% endblock %} +

+
+ +
+ Timeline + Total +
+ +
+ {% for slug, label, url in possible_stats_types %} + {{ label }} + {% endfor %} +
+ +
+ {% for slug, label, url in possible_docs_types %} + {{ label }} + {% endfor %} +
+
+ + +
+
+

+ This page provides a timeline of IETF {{ doc_type|upper }} documents {{ objects }} by {{ stats_type }}. Only the top-{{ top_n }} categories are listed, + the remaining ones are aggregated into 'Other'. The year is the actual publication date for a RFC and the + submission date of the latest revision for a draft (i.e., only the authors of latest revision). +

+
+
+

{{ doc_type|title }} {{ objects|title }} by {{ stats_type|title }}

+
+ +
+
+
+
+

+ Specific lines can be removed by clicking on their legend at the bottom of the graphic. + Hold Alt (or ⌥ on Mac) and scroll/drag to zoom & pan. + Panning can be done via the mouse or with a finger. + Zooming is done via the mouse wheel or via a pinch gesture. + Press ESC + or click to reset panning/zooming. +

+
+{% endblock %} diff --git a/ietf/templates/stats/documents_total.html b/ietf/templates/stats/documents_total.html new file mode 100644 index 00000000000..10361190488 --- /dev/null +++ b/ietf/templates/stats/documents_total.html @@ -0,0 +1,75 @@ +{% extends "base.html" %} +{% load origin %} +{% load ietf_filters static django_bootstrap5 %} +{% block js %} + {{ chart_data|json_script:"chart_data" }} + {{ objects|json_script:"objects" }} + +{% endblock %} +{% block content %} + {% origin %} +

+ {% block title %} + Statistics for IETF {{ objects|title }} + {% endblock %} +

+
+ +
+ Timeline + Total +
+ +
+ {% for slug, label, url in possible_stats_types %} + {{ label }} + {% endfor %} +
+ +
+ {% for slug, label, url in possible_docs_types %} + {{ label }} + {% endfor %} +
+
+ + +
+
+

+ This page provides the top-{{ top_n }} {{ stats_type }} for IETF {{ doc_type|upper }} {{ objects}}. + Only the top-{{ top_n }} categories are listed, the remaining ones are aggregated into 'Other'. + {% if objects == 'authors' %} + Note: authors are counted only once no matter how many documents they have authored. + {% endif %} +

+
+
+

{{ doc_type|title }} {{ objects|title }} by {{ stats_type|title }}

+
+ +
+
+
+
+

+ Click on a bar to hide it and rescale the graph. +

+
+{% endblock %} diff --git a/ietf/templates/stats/error.html b/ietf/templates/stats/error.html new file mode 100644 index 00000000000..d9738a32220 --- /dev/null +++ b/ietf/templates/stats/error.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} +{% load origin %} +{% load ietf_filters static %} +{% block content %} + {% origin %} +

+ {% block title %}Invalid input{% endblock %} +

+

+ There was an error in your request: {{ message }}. +

+{% endblock %} diff --git a/ietf/templates/stats/index.html b/ietf/templates/stats/index.html index 38c8069507f..c77bd3334ba 100644 --- a/ietf/templates/stats/index.html +++ b/ietf/templates/stats/index.html @@ -11,17 +11,23 @@

-

- Statistics on authorship are not currently available. -

-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/templates/stats/known_countries_list.html b/ietf/templates/stats/known_countries_list.html index 0cdac6b6dfa..8c5157eeded 100644 --- a/ietf/templates/stats/known_countries_list.html +++ b/ietf/templates/stats/known_countries_list.html @@ -47,4 +47,4 @@

{% endblock %} {% block js %} - {% endblock %} \ No newline at end of file + {% endblock %} diff --git a/ietf/templates/stats/meeting_stats.html b/ietf/templates/stats/meeting_stats.html index fc41949a2ef..5150785708c 100644 --- a/ietf/templates/stats/meeting_stats.html +++ b/ietf/templates/stats/meeting_stats.html @@ -5,7 +5,7 @@ {% block js %} {{ total_chart_data|json_script:"total-chart-data" }} {{ in_person_chart_data|json_script:"in-person-chart-data" }} - + {% endblock %} {% block content %} {% origin %} @@ -22,7 +22,7 @@

{% if slug == stats_type %} active {% endif %}" - href="{{ url }}">{{ label }} + href="{{ url }}?top={{ top_n }}">{{ label }} {% endfor %}

@@ -32,27 +32,56 @@

{% if num == meeting_number %} active {% endif %}" - href="{{ url }}">{{ num }} + href="{{ url }}?top={{ top_n }}">{{ num }} {% endfor %} +
+ + +

- This page provides a visual representation of the total registrations for IETF-{{ meeting_number }} by {{ stats_type }}. - Only categories having more than {{ minimum_required }} registrations are displayed separately, + This page provides a visual representation of the total registrations for IETF-{{ meeting_number }} by {{ stats_type }}. + Only the top-{{ top_n }} {{ stats_type }} registrations are displayed separately, else they are grouped under "Other".

-

Total Registrations by {{ stats_type|title }} ({{ total_total}} in total)

+

+ Total Registrations by {{ stats_type|title }} ({{ total_total}} in total) + + + +

-

In Person Registrations by {{ stats_type|title }} ({{ in_person_total}} in total)

+

+ In Person Registrations by {{ stats_type|title }} ({{ in_person_total}} in total) + + + +

-{% endblock %} \ No newline at end of file +
+

+ Hover over a slice to see the count and percentage of total registrations. + Click on a legend at the bottom to hide it and rescale the graph; click again to show it. + Click on to download the data as a CSV file. +

+
+{% endblock %} diff --git a/ietf/templates/stats/meetings_timeline.html b/ietf/templates/stats/meetings_timeline.html index 40f46880ccf..f20efa74949 100644 --- a/ietf/templates/stats/meetings_timeline.html +++ b/ietf/templates/stats/meetings_timeline.html @@ -6,7 +6,7 @@ {{ total_chart_data|json_script:"total-chart-data" }} {{ in_person_chart_data|json_script:"in-person-chart-data" }} {{ stats_type|json_script:"stats-type-data" }} - + {% endblock %} {% block content %} {% origin %} @@ -24,51 +24,71 @@

{% if slug == stats_type %} active {% endif %}" - href="{{ url }}">{{ label }} + href="{{ url }}?top={{ top_n }}">{{ label }} {% endfor %}
{% for num, url in possible_meeting_numbers %} {{ num }} + href="{{ url }}?top={{ top_n }}">{{ num }} {% endfor %}
+
+ + +

- {% if stats_type == 'total' %} + {% if stats_type == 'reg_type' %} This page provides a timeline of meeting registrations. {% else %} - This page provides a timeline of meeting registrations by {{ stats_type }} with a limit of {{ top_n }} categories. + This page provides a timeline of meeting registrations by {{ stats_type }} with a limit of {{ top_n }} {{ plural_stats_type }}. {% endif %} - Panning can be done via the mouse or with a finger. Zooming is done via the mouse wheel or via a pinch gesture. Press ESC - or click to reset panning/zooming.

- {% if stats_type == 'total' %} -

Total Registrations

- {% else %} -

Total Registrations by {{ stats_type|title }}

- {% endif %} +

Total Registrations by {{ stats_type|title }} + + + +

- {% if stats_type != 'total' %} + {% if stats_type != 'reg_type' %}
- {% if stats_type == 'total' %} -

Total In Person Registrations

- {% else %} -

In Person Registrations by {{ stats_type|title }}

- {% endif %} +

In Person Registrations by {{ stats_type|title }} + + + +

{% endif %}
-{% endblock %} \ No newline at end of file +
+

+ Specific lines can be removed by clicking on their legend at the bottom of the graphic. + Hold Alt (or ⌥ on Mac) and scroll/drag to zoom & pan. + Panning can be done via the mouse or with a finger. + Zooming is done via the mouse wheel or via a pinch gesture. Press ESC + or click to reset panning/zooming. + Click on to download the data as a CSV file. +

+
+{% endblock %} diff --git a/ietf/templates/stats/review_stats.html b/ietf/templates/stats/review_stats.html index 2741f165182..3c6505c2dac 100644 --- a/ietf/templates/stats/review_stats.html +++ b/ietf/templates/stats/review_stats.html @@ -365,4 +365,4 @@

}); {% endif %} - {% endblock %} \ No newline at end of file + {% endblock %} diff --git a/ietf/templates/stats/used_affiliations_list.html b/ietf/templates/stats/used_affiliations_list.html new file mode 100644 index 00000000000..e5a382c4d04 --- /dev/null +++ b/ietf/templates/stats/used_affiliations_list.html @@ -0,0 +1,37 @@ +{% extends "base.html" %} +{% load origin %} +{% load ietf_filters static %} +{% block pagehead %} + +{% endblock %} +{% block content %} + {% origin %} +

+ {% block title %}Used Affiliations in IETF Drafts{% endblock %} +

+

+ In case you think an affiliation mapping is wrong from the list, you can + file a ticket. +

+ + + + + + + + + + {% for a in affiliations %} + + + + + + {% endfor %} + +
Affiliation in IETF DraftsNumber of OccurrencesCanonicalised Affiliation
{{ a.affiliation }}{{ a.author_count }}{{ a.canonical }}
+ {% endblock %} + {% block js %} + + {% endblock %} diff --git a/package.json b/package.json index 008fd57e7fb..55c67b40a0e 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,6 @@ "browser-fs-access": "0.38.0", "caniuse-lite": "1.0.30001803", "chart.js": "^4.5.1", - "chartjs-plugin-autocolors": "0.3.1", "chartjs-plugin-zoom": "2.2.0", "d3": "7.9.0", "file-saver": "2.0.5", @@ -155,8 +154,10 @@ "ietf/static/js/manage-community-list.js", "ietf/static/js/manage-review-requests.js", "ietf/static/js/meeting-interim-request.js", - "ietf/static/js/meeting_stats.js", - "ietf/static/js/meeting_timeline.js", + "ietf/static/js/stats_document_timeline.js", + "ietf/static/js/stats_document_total.js", + "ietf/static/js/stats_meeting.js", + "ietf/static/js/stats_meeting_timeline.js", "ietf/static/js/moment.js", "ietf/static/js/navbar-doc-search.js", "ietf/static/js/password_strength.js",