feat: self-serve queries for inputs to reporting and survey purposes

ietf/ietfauth/utils.py

@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2013-2022, All Rights Reserved
+# Copyright The IETF Trust 2013-2026, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -163,6 +163,7 @@ def has_role(user, role_names, *args, **kwargs):
                 | Q(name="atlarge", group__acronym="irsg")
             ),
             "RSAB Member": Q(name="member", group__acronym="rsab"),
+            "LLC Staff": Q(name="member", group__acronym="llc-staff"),
             "Robot": Q(name="robot", group__acronym="secretariat"),
         }
 

ietf/stats/tests.py

@@ -1,8 +1,10 @@
 # Copyright The IETF Trust 2016-2026, All Rights Reserved
 
 import calendar
-import json
+import csv
 import datetime
+import io
+import json
 
 from django.http import Http404
 from pyquery import PyQuery
@@ -18,10 +20,12 @@
 import ietf.stats.views
 
 
-from ietf.group.factories import RoleFactory
-from ietf.person.factories import PersonFactory
+from ietf.doc.factories import NewRevisionDocEventFactory
+from ietf.group.factories import GroupFactory, RoleFactory
+from ietf.person.factories import EmailFactory, PersonFactory
 from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory
 from ietf.meeting.tests_models import MeetingFactory, RegistrationFactory
+from ietf.submit.factories import SubmissionFactory
 from ietf.utils.timezone import date_today
 
 
@@ -183,3 +187,139 @@ def test_review_stats(self):
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertTrue(q('.review-stats td:contains("1")'))
+
+
+class AnnualReportInputsTests(TestCase):
+    def setUp(self):
+        super().setUp()
+        llc_staff = GroupFactory(acronym="llc-staff", type_id="team")
+        self.member = PersonFactory()
+        RoleFactory(group=llc_staff, name_id="member", person=self.member)
+        self.non_member = PersonFactory()
+
+    def _member_login(self):
+        self.client.login(
+            username=self.member.user.username,
+            password=f"{self.member.user.username}+password",
+        )
+
+    def test_access_unauthenticated(self):
+        url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"})
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 302)
+        self.assertIn("/accounts/login", r["Location"])
+
+    def test_access_non_member_forbidden(self):
+        url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"})
+        self.client.login(
+            username=self.non_member.user.username,
+            password=f"{self.non_member.user.username}+password",
+        )
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 403)
+
+    def test_access_member_allowed(self):
+        self._member_login()
+        url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"})
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+
+    def test_default_year(self):
+        self._member_login()
+        url = urlreverse(ietf.stats.views.annual_report_inputs)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.context["year"], datetime.date.today().year - 1)
+
+    def test_year_param_redirects_to_year_url(self):
+        self._member_login()
+        url = urlreverse(ietf.stats.views.annual_report_inputs)
+        r = self.client.get(url, {"year": "2022"})
+        self.assertRedirects(
+            r,
+            urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2022"}),
+        )
+
+    def test_summary_counts(self):
+        self._member_login()
+        year = 2021
+        # author1 has a matching Person record; author2 does not
+        EmailFactory(address="author1@example.com")
+        sub = SubmissionFactory(
+            state_id="posted",
+            submission_date=datetime.date(year, 6, 1),
+            submitter_email="submitter@example.com",
+        )
+        sub.authors = [
+            {"name": "Author One", "email": "author1@example.com", "affiliation": "", "country": "", "errors": []},
+            {"name": "Author Two", "email": "author2@example.com", "affiliation": "", "country": "", "errors": []},
+        ]
+        sub.save()
+        NewRevisionDocEventFactory(
+            time=datetime.datetime(year, 6, 1, tzinfo=datetime.timezone.utc),
+            doc__type_id="draft",
+        )
+        NewRevisionDocEventFactory(
+            time=datetime.datetime(year, 6, 1, tzinfo=datetime.timezone.utc),
+            doc__type_id="draft",
+        )
+        # Same draft, second revision — should count once
+        extra = NewRevisionDocEventFactory(
+            time=datetime.datetime(year, 9, 1, tzinfo=datetime.timezone.utc),
+            doc__type_id="draft",
+        )
+        NewRevisionDocEventFactory(
+            time=datetime.datetime(year, 9, 15, tzinfo=datetime.timezone.utc),
+            doc=extra.doc,
+        )
+
+        url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)})
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.context["year"], year)
+        self.assertEqual(r.context["author_count"], 2)
+        self.assertEqual(r.context["author_person_count"], 1)
+        self.assertEqual(r.context["author_noperson_count"], 1)
+        self.assertEqual(r.context["submitter_count"], 1)
+        self.assertEqual(r.context["submitter_person_count"], 0)
+        self.assertEqual(r.context["submitter_noperson_count"], 1)
+        self.assertEqual(r.context["draft_count"], 3)
+
+    def test_download_authors_csv(self):
+        self._member_login()
+        year = 2020
+        sub = SubmissionFactory(
+            state_id="posted",
+            submission_date=datetime.date(year, 4, 1),
+        )
+        sub.authors = [
+            {"name": "Author", "email": "csvauthor@example.com", "affiliation": "", "country": "", "errors": []},
+        ]
+        sub.save()
+
+        url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)})
+        r = self.client.get(url, {"download": "authors"})
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r["Content-Type"], "text/csv")
+        self.assertIn(f"authors-{year}.csv", r["Content-Disposition"])
+        rows = list(csv.reader(io.StringIO(r.content.decode())))
+        self.assertEqual(len(rows), 1)
+        self.assertIn("csvauthor@example.com", rows[0])
+
+    def test_download_submitters_csv(self):
+        self._member_login()
+        year = 2020
+        SubmissionFactory(
+            state_id="posted",
+            submission_date=datetime.date(year, 4, 1),
+            submitter_email="csvsubmitter@example.com",
+        )
+
+        url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)})
+        r = self.client.get(url, {"download": "submitters"})
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r["Content-Type"], "text/csv")
+        self.assertIn(f"submitters-{year}.csv", r["Content-Disposition"])
+        rows = list(csv.reader(io.StringIO(r.content.decode())))
+        self.assertEqual(len(rows), 1)
+        self.assertIn("csvsubmitter@example.com", rows[0])

ietf/stats/urls.py

@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2016-2020, All Rights Reserved
+# Copyright The IETF Trust 2016-2026, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -15,4 +15,5 @@
     url(r"^meeting/(?P<meeting_number>\d+)/(?P<stats_type>affiliation|country)/$", views.meeting_stats),
     url(r"^meeting/(?:(?P<stats_type>affiliation|country|total)/)?$", views.meetings_timeline),
     url(r"^review/(?:(?P<stats_type>completion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats),
+    url(r"^annual_report_inputs/(?:(?P<year>\d{4})/)?$", views.annual_report_inputs),
 ]

ietf/stats/views.py

@@ -1,8 +1,9 @@
-# Copyright The IETF Trust 2016-2020, All Rights Reserved
+# Copyright The IETF Trust 2016-2026, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
 import calendar
+import csv
 import datetime
 import itertools
 import json
@@ -12,7 +13,7 @@
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.core.cache import cache
-from django.http import HttpResponseRedirect
+from django.http import HttpResponse, HttpResponseRedirect
 from django.shortcuts import render, get_object_or_404
 from django.urls import reverse as urlreverse
 from django.db.models import Count
@@ -28,7 +29,7 @@
 from ietf.person.models import Person
 from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName
 from ietf.meeting.models import Registration, Meeting
-from ietf.ietfauth.utils import has_role
+from ietf.ietfauth.utils import has_role, role_required
 from ietf.utils.response import permission_denied
 from ietf.utils.timezone import date_today, DEADLINE_TZINFO
 from ietf.meeting.helpers import get_current_ietf_meeting_num
@@ -957,3 +958,46 @@ def time_key_fn(t):
         "possible_states": possible_states,
         "selected_state": selected_state,
     })
+
+
+@role_required("LLC Staff")
+def annual_report_inputs(request, year=None):
+    if year is None and "year" in request.GET:
+        return HttpResponseRedirect(
+            urlreverse("ietf.stats.views.annual_report_inputs", kwargs={"year": request.GET["year"]})
+        )
+    year = int(year) if year else datetime.date.today().year - 1
+
+    from ietf.doc.models import NewRevisionDocEvent
+    from ietf.utils.reports import authors_by_year, submitters_by_year, unique_people
+
+    download = request.GET.get("download")
+    if download in ("authors", "submitters"):
+        addresses = authors_by_year(year) if download == "authors" else submitters_by_year(year)
+        response = HttpResponse(content_type="text/csv")
+        response["Content-Disposition"] = f'attachment; filename="{download}-{year}.csv"'
+        writer = csv.writer(response)
+        writer.writerow(sorted(addresses))
+        return response
+
+    authors = authors_by_year(year)
+    submitters = submitters_by_year(year)
+    author_persons, author_nopersons = unique_people(authors)
+    submitter_persons, submitter_nopersons = unique_people(submitters)
+
+    draft_count = len(set(
+        NewRevisionDocEvent.objects.filter(
+            doc__type_id="draft", time__year=year
+        ).values_list("doc__name", flat=True)
+    ))
+
+    return render(request, "stats/annual_report_inputs.html", {
+        "year": year,
+        "author_count": len(authors),
+        "submitter_count": len(submitters),
+        "author_person_count": author_persons.count(),
+        "author_noperson_count": len(author_nopersons),
+        "submitter_person_count": submitter_persons.count(),
+        "submitter_noperson_count": len(submitter_nopersons),
+        "draft_count": draft_count,
+    })

ietf/submit/factories.py

@@ -21,6 +21,9 @@ class Meta:
 
 class SubmissionFactory(factory.django.DjangoModelFactory):
     state_id = 'uploaded'
+    submitter_name = factory.Faker("name")
+    submitter_email = factory.Faker("email") 
+    submitter = factory.LazyAttribute(lambda o: f"{o.submitter_name} <{o.submitter_email}>")
 
     @factory.lazy_attribute_sequence
     def name(self, n):
@@ -32,3 +35,4 @@ def auth_key(self):
 
     class Meta:
         model = Submission
+        exclude = ("submitter_name", "submitter_email")

ietf/templates/base/menu.html

@@ -1,4 +1,4 @@
-{# Copyright The IETF Trust 2015-2025, All Rights Reserved #}
+{# Copyright The IETF Trust 2015-2026, All Rights Reserved #}
 {% load origin %}
 {% origin %}
 {% load ietf_filters managed_groups wg_menu active_groups_menu group_filters cache meetings_filters %}
@@ -453,6 +453,14 @@
                     </a>
                 </li>
             {% endif %}
+            {% if user|has_role:"LLC Staff" %}
+                <li>
+                    <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
+                       href="{% url "ietf.stats.views.annual_report_inputs" %}">
+                        Annual report inputs
+                    </a>
+                </li>
+            {% endif %}
         </ul>
     </li>
     <li>

ietf/templates/stats/annual_report_inputs.html

@@ -0,0 +1,55 @@
+{# Copyright The IETF Trust 2026, All Rights Reserved #}
+{% extends "base.html" %}
+{% load origin %}
+{% load ietf_filters static %}
+{% block content %}
+    {% origin %}
+    <h1>{% block title %}Annual Report Inputs for {{ year }}{% endblock %}</h1>
+    <form method="get" action="{% url "ietf.stats.views.annual_report_inputs" %}" class="mb-3">
+        <div class="input-group w-auto d-inline-flex">
+            <label class="input-group-text" for="year-select">Year</label>
+            <input id="year-select" class="form-control" type="number" name="year" value="{{ year }}" min="1990">
+            <button class="btn btn-primary" type="submit">Go</button>
+        </div>
+    </form>
+    <h2>Summary</h2>
+    <table class="table table-sm table-bordered w-auto">
+        <thead>
+            <tr>
+                <th scope="col" class="px-3"></th>
+                <th scope="col" class="text-end px-3">Email addresses</th>
+                <th scope="col" class="text-end px-3">Unique persons found</th>
+                <th scope="col" class="text-end px-3">Addresses with no person record</th>
+                <th scope="col" class="text-end px-3">Approximate unique people</th>
+            </tr>
+        </thead>
+        <tbody>
+            <tr>
+                <th scope="row" class="px-3">Authors</th>
+                <td class="text-end px-3">{{ author_count }}</td>
+                <td class="text-end px-3">{{ author_person_count }}</td>
+                <td class="text-end px-3">{{ author_noperson_count }}</td>
+                <td class="text-end px-3">{{ author_person_count|add:author_noperson_count }}</td>
+            </tr>
+            <tr>
+                <th scope="row" class="px-3">Submitters</th>
+                <td class="text-end px-3">{{ submitter_count }}</td>
+                <td class="text-end px-3">{{ submitter_person_count }}</td>
+                <td class="text-end px-3">{{ submitter_noperson_count }}</td>
+                <td class="text-end px-3">{{ submitter_person_count|add:submitter_noperson_count }}</td>
+            </tr>
+        </tbody>
+    </table>
+    <p>Drafts submitted in {{ year }}: {{ draft_count }}</p>
+    <h2>Downloads</h2>
+    <ul>
+        <li>
+            <a href="{% url "ietf.stats.views.annual_report_inputs" year %}?download=authors">authors-{{ year }}.csv</a>
+            &mdash; email addresses for I-D authors with submissions posted in {{ year }}
+        </li>
+        <li>
+            <a href="{% url "ietf.stats.views.annual_report_inputs" year %}?download=submitters">submitters-{{ year }}.csv</a>
+            &mdash; email addresses for I-D submitters with submissions posted in {{ year }}
+        </li>
+    </ul>
+{% endblock %}