From f31cd944a21e7a73ec8766e4444eb6d618c4651c Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 6 Jan 2023 14:30:22 -0600 Subject: [PATCH 01/10] feat: utilities to count I-D submitters and authors by year --- ietf/submit/factories.py | 4 + ietf/utils/reports.py | 51 ++++++++++ ietf/utils/tests_reports.py | 185 ++++++++++++++++++++++++++++++++++++ 3 files changed, 240 insertions(+) create mode 100755 ietf/utils/reports.py create mode 100755 ietf/utils/tests_reports.py diff --git a/ietf/submit/factories.py b/ietf/submit/factories.py index 1076434b82..69888a055d 100644 --- a/ietf/submit/factories.py +++ b/ietf/submit/factories.py @@ -21,6 +21,9 @@ class Meta: class SubmissionFactory(factory.django.DjangoModelFactory): state_id = 'uploaded' + submitter_name = factory.Faker("name") + submitter_email = factory.Faker("email") + submitter = factory.LazyAttribute(lambda o: f"{o.submitter_name} <{o.submitter_email}>") @factory.lazy_attribute_sequence def name(self, n): @@ -32,3 +35,4 @@ def auth_key(self): class Meta: model = Submission + exclude = ("submitter_name", "submitter_email") diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py new file mode 100755 index 0000000000..4580509042 --- /dev/null +++ b/ietf/utils/reports.py @@ -0,0 +1,51 @@ +# Copyright The IETF Trust 2023, All Rights Reserved + +from typing import List, Tuple + +from email.utils import parseaddr + +from ietf.person.models import Person +from ietf.submit.models import Submission + + +def authors_by_year(year: int) -> List[str]: + """ + Returns the email addresses provided by I-D authors for + drafts that were submitted in the given year. + """ + addresses = set() + for submission in Submission.objects.filter(submission_date__year=year): + addresses.update([a["email"] for a in submission.authors]) + return list(addresses) + + +def submitters_by_year(year: int) -> List[str]: + """ + Returns the email addresses provided by I-D submitters for + drafts that were submitted in the given year. + """ + return list( + set( + [ + parseaddr(a)[1] + for a in Submission.objects.filter( + submitter__contains="@", submission_date__year=year + ).values_list("submitter", flat=True) + ] + ) + ) + + +def unique_people(addresses: List[str]) -> Tuple[List, List]: + """ + Given a list of email addresses, return + ( + a list of unique Person records with a matching email address, + a list of unique email addresses with no matching Person record + ) + The sum of the lenghts of these lists is a best-approximation for how + many unique people the list of addresses belong to. + """ + persons = Person.objects.filter(email__address__in=addresses).distinct() + known_email = set(persons.values_list("email__address", flat=True)) + return (list(persons), list(set(addresses) - set(known_email))) diff --git a/ietf/utils/tests_reports.py b/ietf/utils/tests_reports.py new file mode 100755 index 0000000000..9be836da8f --- /dev/null +++ b/ietf/utils/tests_reports.py @@ -0,0 +1,185 @@ +# Copyright The IETF Trust 2023, All Rights Reserved + +import datetime + +import debug # pyflakes:ignore + +from ietf.doc.factories import IndividualDraftFactory +from ietf.person.factories import EmailFactory +from ietf.person.models import Person, Email +from ietf.submit.factories import SubmissionFactory +from ietf.utils.reports import authors_by_year, submitters_by_year, unique_people +from ietf.utils.test_utils import TestCase + + +class ReportTests(TestCase): + def setUp(self): + super().setUp() + + # Build 5 drafts submitted across two years, with 6 unique authors, + # one of which has more than one email address (author0@example.com and + # author0@example.net). The drafts are submitted by two of the six authors, + # again using multiple addresses for author0, and two identities that are not authors. + # Then build a draft where the submission's submitter info doesn't contain an email + # address (we have those in the production database) to make sure that submitter isn't + # counted. + + self.make_draft_submission( + year=2020, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 1", "author1@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=3, + day=3, + submitter_name="NotanAuthor 0", + submitter_email="notanauthor0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.com"), # Note alternate email + ("Author 3", "author3@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=12, + day=31, + submitter_name="Author 3", + submitter_email="author3@example.net", + author_nameaddrs=[("Author 3", "author3@example.net")], + ) + + self.make_draft_submission( + year=2021, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.com", # Note alternate email + author_nameaddrs=[ + ("Author 0", "author0@example.com"), + ("Author 4", "author4@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="NoatanAuthor 2", + submitter_email="notanauthor2@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 3", "author3@example.net"), + ("Author 5", "author5@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="Trouble Maker", + submitter_email="", + author_nameaddrs=[("Author 0", "author0@example.net")], + ) + + def make_draft_submission( + self, year, month, day, submitter_name, submitter_email, author_nameaddrs + ): + + authors = [] + for name, addr in author_nameaddrs: + person = Person.objects.filter(name=name).first() + if not person: + person = EmailFactory(person__name=name, address=addr).person + elif not Email.objects.filter(address=addr).exists(): + EmailFactory(person=person, address=addr) + authors.append(person) + + submission = SubmissionFactory( + submission_date=datetime.date(year, month, day), + submitter_name=submitter_name, + submitter_email=submitter_email, + state_id="posted", + ) + submission.authors = [ + { + "name": f"{name}", + "email": f"{addr}", + "affiliation": "", + "country": "", + "errors": [], + } + for name, addr in author_nameaddrs + ] + + submission.save() + IndividualDraftFactory(submission=submission, authors=authors) + + def test_authors_by_year(self): + authors2020 = authors_by_year(2020) + self.assertEqual( + set(authors2020), + set( + [ + "author0@example.net", + "author0@example.com", + "author1@example.net", + "author3@example.net", + ] + ), + ) + authors2021 = authors_by_year(2021) + self.assertEqual( + set(authors2021), + set( + [ + "author0@example.net", + "author0@example.com", + "author3@example.net", + "author4@example.net", + "author5@example.net", + ] + ), + ) + + def test_submitters_by_year(self): + sub2020 = submitters_by_year(2020) + self.assertEqual( + set(sub2020), + set( + [ + "author0@example.net", + "author3@example.net", + "notanauthor0@example.net", + ] + ), + ) + sub2021 = submitters_by_year(2021) + self.assertEqual( + set(sub2021), set(["author0@example.com", "notanauthor2@example.net"]) + ) + + def test_unique_people(self): + persons, addrs = unique_people( + [ + "notanauthor0@example.com", + "author0@example.net", + "author0@example.com", + "author1@example.net", + "notanauthor0@example.com", + ] + ) + self.assertEqual(addrs, ["notanauthor0@example.com"]) + self.assertEqual( + set(persons), set(Person.objects.filter(name__in=("Author 0", "Author 1"))) + ) + self.assertEqual(len(persons) + len(addrs), 3) From 8c911113c0f99416ae28e1e955312aaf62e4262b Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 6 Jan 2023 14:40:35 -0600 Subject: [PATCH 02/10] chore: fix comment typo --- ietf/utils/reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py index 4580509042..c5bed2b32f 100755 --- a/ietf/utils/reports.py +++ b/ietf/utils/reports.py @@ -43,7 +43,7 @@ def unique_people(addresses: List[str]) -> Tuple[List, List]: a list of unique Person records with a matching email address, a list of unique email addresses with no matching Person record ) - The sum of the lenghts of these lists is a best-approximation for how + The sum of the lengths of these lists is a best-approximation for how many unique people the list of addresses belong to. """ persons = Person.objects.filter(email__address__in=addresses).distinct() From f998f5af6a36bc69531bf1cf9b99655879e421d5 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 9 Jan 2023 10:52:55 -0600 Subject: [PATCH 03/10] fix: return querysets and sets rather than lists. Improve docstrings. --- ietf/utils/reports.py | 20 ++++++++------------ ietf/utils/tests_reports.py | 10 +++++++--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py index c5bed2b32f..f5e5a077e1 100755 --- a/ietf/utils/reports.py +++ b/ietf/utils/reports.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2023, All Rights Reserved -from typing import List, Tuple +from typing import List, Set, Tuple +from django.db.models import QuerySet from email.utils import parseaddr @@ -9,10 +10,7 @@ def authors_by_year(year: int) -> List[str]: - """ - Returns the email addresses provided by I-D authors for - drafts that were submitted in the given year. - """ + """Email addresses provided by I-D authors for drafts that were submitted in the given year.""" addresses = set() for submission in Submission.objects.filter(submission_date__year=year): addresses.update([a["email"] for a in submission.authors]) @@ -20,10 +18,7 @@ def authors_by_year(year: int) -> List[str]: def submitters_by_year(year: int) -> List[str]: - """ - Returns the email addresses provided by I-D submitters for - drafts that were submitted in the given year. - """ + """Email addresses provided by I-D submitters for drafts that were submitted in the given year.""" return list( set( [ @@ -36,8 +31,9 @@ def submitters_by_year(year: int) -> List[str]: ) -def unique_people(addresses: List[str]) -> Tuple[List, List]: - """ +def unique_people(addresses: List[str]) -> Tuple['QuerySet[Person]', Set]: + """Identify Person records matching email addresses and email addresses with no Person record. + Given a list of email addresses, return ( a list of unique Person records with a matching email address, @@ -48,4 +44,4 @@ def unique_people(addresses: List[str]) -> Tuple[List, List]: """ persons = Person.objects.filter(email__address__in=addresses).distinct() known_email = set(persons.values_list("email__address", flat=True)) - return (list(persons), list(set(addresses) - set(known_email))) + return (persons, set(addresses) - set(known_email)) diff --git a/ietf/utils/tests_reports.py b/ietf/utils/tests_reports.py index 9be836da8f..69ac73ce40 100755 --- a/ietf/utils/tests_reports.py +++ b/ietf/utils/tests_reports.py @@ -64,7 +64,7 @@ def setUp(self): submitter_name="Author 0", submitter_email="author0@example.com", # Note alternate email author_nameaddrs=[ - ("Author 0", "author0@example.com"), + ("Author 0", "author0@example.com"), # Again, alternate email ("Author 4", "author4@example.net"), ], ) @@ -88,7 +88,10 @@ def setUp(self): day=31, submitter_name="Trouble Maker", submitter_email="", - author_nameaddrs=[("Author 0", "author0@example.net")], + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 2", "author2@example.net"), + ], ) def make_draft_submission( @@ -144,6 +147,7 @@ def test_authors_by_year(self): [ "author0@example.net", "author0@example.com", + "author2@example.net", "author3@example.net", "author4@example.net", "author5@example.net", @@ -178,7 +182,7 @@ def test_unique_people(self): "notanauthor0@example.com", ] ) - self.assertEqual(addrs, ["notanauthor0@example.com"]) + self.assertEqual(addrs, set(["notanauthor0@example.com"])) self.assertEqual( set(persons), set(Person.objects.filter(name__in=("Author 0", "Author 1"))) ) From 488e024b6fce076c8e7825358fe1e709c4d2213b Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 9 Jan 2023 16:24:00 -0600 Subject: [PATCH 04/10] refactor: don't flatten sets to lists --- ietf/utils/reports.py | 24 +++++++++++------------- ietf/utils/tests_reports.py | 10 +++++----- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py index f5e5a077e1..3798a6f1d9 100755 --- a/ietf/utils/reports.py +++ b/ietf/utils/reports.py @@ -9,29 +9,27 @@ from ietf.submit.models import Submission -def authors_by_year(year: int) -> List[str]: +def authors_by_year(year: int) -> Set[str]: """Email addresses provided by I-D authors for drafts that were submitted in the given year.""" addresses = set() for submission in Submission.objects.filter(submission_date__year=year): addresses.update([a["email"] for a in submission.authors]) - return list(addresses) + return addresses -def submitters_by_year(year: int) -> List[str]: +def submitters_by_year(year: int) -> Set[str]: """Email addresses provided by I-D submitters for drafts that were submitted in the given year.""" - return list( - set( - [ - parseaddr(a)[1] - for a in Submission.objects.filter( - submitter__contains="@", submission_date__year=year - ).values_list("submitter", flat=True) - ] - ) + return set( + [ + parseaddr(a)[1] + for a in Submission.objects.filter( + submitter__contains="@", submission_date__year=year + ).values_list("submitter", flat=True) + ] ) -def unique_people(addresses: List[str]) -> Tuple['QuerySet[Person]', Set]: +def unique_people(addresses: List[str]) -> Tuple["QuerySet[Person]", Set]: """Identify Person records matching email addresses and email addresses with no Person record. Given a list of email addresses, return diff --git a/ietf/utils/tests_reports.py b/ietf/utils/tests_reports.py index 69ac73ce40..4fa27eed70 100755 --- a/ietf/utils/tests_reports.py +++ b/ietf/utils/tests_reports.py @@ -64,7 +64,7 @@ def setUp(self): submitter_name="Author 0", submitter_email="author0@example.com", # Note alternate email author_nameaddrs=[ - ("Author 0", "author0@example.com"), # Again, alternate email + ("Author 0", "author0@example.com"), # Again, alternate email ("Author 4", "author4@example.net"), ], ) @@ -130,7 +130,7 @@ def make_draft_submission( def test_authors_by_year(self): authors2020 = authors_by_year(2020) self.assertEqual( - set(authors2020), + authors2020, set( [ "author0@example.net", @@ -142,7 +142,7 @@ def test_authors_by_year(self): ) authors2021 = authors_by_year(2021) self.assertEqual( - set(authors2021), + authors2021, set( [ "author0@example.net", @@ -158,7 +158,7 @@ def test_authors_by_year(self): def test_submitters_by_year(self): sub2020 = submitters_by_year(2020) self.assertEqual( - set(sub2020), + sub2020, set( [ "author0@example.net", @@ -169,7 +169,7 @@ def test_submitters_by_year(self): ) sub2021 = submitters_by_year(2021) self.assertEqual( - set(sub2021), set(["author0@example.com", "notanauthor2@example.net"]) + sub2021, set(["author0@example.com", "notanauthor2@example.net"]) ) def test_unique_people(self): From 7b3c35f70f858b9778aef75dbd0d67f2f4529d45 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 25 Jan 2024 11:42:07 -0600 Subject: [PATCH 05/10] fix: filter to posted submissions and produce report --- ietf/utils/reports.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py index 3798a6f1d9..e8a2529df7 100755 --- a/ietf/utils/reports.py +++ b/ietf/utils/reports.py @@ -1,5 +1,6 @@ # Copyright The IETF Trust 2023, All Rights Reserved +import csv from typing import List, Set, Tuple from django.db.models import QuerySet @@ -12,7 +13,7 @@ def authors_by_year(year: int) -> Set[str]: """Email addresses provided by I-D authors for drafts that were submitted in the given year.""" addresses = set() - for submission in Submission.objects.filter(submission_date__year=year): + for submission in Submission.objects.filter(submission_date__year=year,state="posted"): addresses.update([a["email"] for a in submission.authors]) return addresses @@ -23,7 +24,7 @@ def submitters_by_year(year: int) -> Set[str]: [ parseaddr(a)[1] for a in Submission.objects.filter( - submitter__contains="@", submission_date__year=year + submitter__contains="@", submission_date__year=year, state="posted" ).values_list("submitter", flat=True) ] ) @@ -43,3 +44,19 @@ def unique_people(addresses: List[str]) -> Tuple["QuerySet[Person]", Set]: persons = Person.objects.filter(email__address__in=addresses).distinct() known_email = set(persons.values_list("email__address", flat=True)) return (persons, set(addresses) - set(known_email)) + +def write_reports(year: int) -> None : + authors = authors_by_year(year) + submitters = submitters_by_year(year) + print(f"authors: {len(authors)}") + print(f"submitters: {len(submitters)}") + persons, nopersons = unique_people(authors) + print(f"authors: unique persons: {len(persons)}, no person found: {len(nopersons)}") + persons, nopersons = unique_people(submitters) + print(f"submitters: unique persons: {len(persons)}, no person found: {len(nopersons)}") + with open("authors.csv","w",newline="") as file: + writer = csv.writer(file) + writer.writerow(authors) + with open("submitters.csv","w",newline="") as file: + writer = csv.writer(file) + writer.writerow(submitters) From c3623de0a56d14d0e6fdc287a2bf6a459901050a Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 25 Jan 2024 11:43:12 -0600 Subject: [PATCH 06/10] chore: black --- ietf/utils/reports.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py index e8a2529df7..5b2b49b273 100755 --- a/ietf/utils/reports.py +++ b/ietf/utils/reports.py @@ -13,7 +13,9 @@ def authors_by_year(year: int) -> Set[str]: """Email addresses provided by I-D authors for drafts that were submitted in the given year.""" addresses = set() - for submission in Submission.objects.filter(submission_date__year=year,state="posted"): + for submission in Submission.objects.filter( + submission_date__year=year, state="posted" + ): addresses.update([a["email"] for a in submission.authors]) return addresses @@ -45,7 +47,8 @@ def unique_people(addresses: List[str]) -> Tuple["QuerySet[Person]", Set]: known_email = set(persons.values_list("email__address", flat=True)) return (persons, set(addresses) - set(known_email)) -def write_reports(year: int) -> None : + +def write_reports(year: int) -> None: authors = authors_by_year(year) submitters = submitters_by_year(year) print(f"authors: {len(authors)}") @@ -53,10 +56,12 @@ def write_reports(year: int) -> None : persons, nopersons = unique_people(authors) print(f"authors: unique persons: {len(persons)}, no person found: {len(nopersons)}") persons, nopersons = unique_people(submitters) - print(f"submitters: unique persons: {len(persons)}, no person found: {len(nopersons)}") - with open("authors.csv","w",newline="") as file: + print( + f"submitters: unique persons: {len(persons)}, no person found: {len(nopersons)}" + ) + with open("authors.csv", "w", newline="") as file: writer = csv.writer(file) writer.writerow(authors) - with open("submitters.csv","w",newline="") as file: + with open("submitters.csv", "w", newline="") as file: writer = csv.writer(file) writer.writerow(submitters) From 11cbfc0a3560d29cf28850fea63c69df2de46806 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 25 Jan 2024 11:46:29 -0600 Subject: [PATCH 07/10] chore: boilerplate --- ietf/utils/reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py index 5b2b49b273..9708f2e8c0 100755 --- a/ietf/utils/reports.py +++ b/ietf/utils/reports.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2023, All Rights Reserved +# Copyright The IETF Trust 2023-2024, All Rights Reserved import csv from typing import List, Set, Tuple From 2968bc9e93ebc9824c9b2964febe95fb8a28d901 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 5 Dec 2025 16:59:08 -0600 Subject: [PATCH 08/10] fix: re-re-re fix the ^M problem in the issue template --- .github/ISSUE_TEMPLATE/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 5e27af9fed..320614b17e 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,8 +1,8 @@ -blank_issues_enabled: false -contact_links: - - name: Help and questions - url: https://github.com/ietf-tools/datatracker/discussions/categories/help-questions - about: Need help? Have a question on setting up the project or its usage? - - name: Discuss new ideas - url: https://github.com/ietf-tools/datatracker/discussions/categories/ideas - about: Submit ideas for new features or improvements to be discussed. +blank_issues_enabled: false +contact_links: + - name: Help and questions + url: https://github.com/ietf-tools/datatracker/discussions/categories/help-questions + about: Need help? Have a question on setting up the project or its usage? + - name: Discuss new ideas + url: https://github.com/ietf-tools/datatracker/discussions/categories/ideas + about: Submit ideas for new features or improvements to be discussed. From de600c64748f3a7f859adae896eb8f424009cc6e Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 2 Jun 2026 14:19:02 -0500 Subject: [PATCH 09/10] feat: self-serve queries for inputs to reporting and survey purposes --- ietf/ietfauth/utils.py | 3 +- ietf/stats/tests.py | 146 +++++++++++++++++- ietf/stats/urls.py | 3 +- ietf/stats/views.py | 50 +++++- ietf/templates/base/menu.html | 10 +- .../templates/stats/annual_report_inputs.html | 55 +++++++ ietf/utils/reports.py | 20 +-- 7 files changed, 259 insertions(+), 28 deletions(-) create mode 100644 ietf/templates/stats/annual_report_inputs.html diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py index 0df667fbd2..30d51cddd0 100644 --- a/ietf/ietfauth/utils.py +++ b/ietf/ietfauth/utils.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved +# Copyright The IETF Trust 2013-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -163,6 +163,7 @@ def has_role(user, role_names, *args, **kwargs): | Q(name="atlarge", group__acronym="irsg") ), "RSAB Member": Q(name="member", group__acronym="rsab"), + "LLC Staff": Q(name="member", group__acronym="llc-staff"), "Robot": Q(name="robot", group__acronym="secretariat"), } diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index dc5b5d6ae8..d8e8741702 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -1,8 +1,10 @@ # Copyright The IETF Trust 2016-2026, All Rights Reserved import calendar -import json +import csv import datetime +import io +import json from django.http import Http404 from pyquery import PyQuery @@ -18,10 +20,12 @@ import ietf.stats.views -from ietf.group.factories import RoleFactory -from ietf.person.factories import PersonFactory +from ietf.doc.factories import NewRevisionDocEventFactory +from ietf.group.factories import GroupFactory, RoleFactory +from ietf.person.factories import EmailFactory, PersonFactory from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory from ietf.meeting.tests_models import MeetingFactory, RegistrationFactory +from ietf.submit.factories import SubmissionFactory from ietf.utils.timezone import date_today @@ -183,3 +187,139 @@ def test_review_stats(self): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('.review-stats td:contains("1")')) + + +class AnnualReportInputsTests(TestCase): + def setUp(self): + super().setUp() + llc_staff = GroupFactory(acronym="llc-staff", type_id="team") + self.member = PersonFactory() + RoleFactory(group=llc_staff, name_id="member", person=self.member) + self.non_member = PersonFactory() + + def _member_login(self): + self.client.login( + username=self.member.user.username, + password=f"{self.member.user.username}+password", + ) + + def test_access_unauthenticated(self): + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + r = self.client.get(url) + self.assertEqual(r.status_code, 302) + self.assertIn("/accounts/login", r["Location"]) + + def test_access_non_member_forbidden(self): + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + self.client.login( + username=self.non_member.user.username, + password=f"{self.non_member.user.username}+password", + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + + def test_access_member_allowed(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + + def test_default_year(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.context["year"], datetime.date.today().year - 1) + + def test_year_param_redirects_to_year_url(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs) + r = self.client.get(url, {"year": "2022"}) + self.assertRedirects( + r, + urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2022"}), + ) + + def test_summary_counts(self): + self._member_login() + year = 2021 + # author1 has a matching Person record; author2 does not + EmailFactory(address="author1@example.com") + sub = SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 6, 1), + submitter_email="submitter@example.com", + ) + sub.authors = [ + {"name": "Author One", "email": "author1@example.com", "affiliation": "", "country": "", "errors": []}, + {"name": "Author Two", "email": "author2@example.com", "affiliation": "", "country": "", "errors": []}, + ] + sub.save() + NewRevisionDocEventFactory( + time=datetime.datetime(year, 6, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + NewRevisionDocEventFactory( + time=datetime.datetime(year, 6, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + # Same draft, second revision — should count once + extra = NewRevisionDocEventFactory( + time=datetime.datetime(year, 9, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + NewRevisionDocEventFactory( + time=datetime.datetime(year, 9, 15, tzinfo=datetime.timezone.utc), + doc=extra.doc, + ) + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.context["year"], year) + self.assertEqual(r.context["author_count"], 2) + self.assertEqual(r.context["author_person_count"], 1) + self.assertEqual(r.context["author_noperson_count"], 1) + self.assertEqual(r.context["submitter_count"], 1) + self.assertEqual(r.context["submitter_person_count"], 0) + self.assertEqual(r.context["submitter_noperson_count"], 1) + self.assertEqual(r.context["draft_count"], 3) + + def test_download_authors_csv(self): + self._member_login() + year = 2020 + sub = SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 4, 1), + ) + sub.authors = [ + {"name": "Author", "email": "csvauthor@example.com", "affiliation": "", "country": "", "errors": []}, + ] + sub.save() + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url, {"download": "authors"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "text/csv") + self.assertIn(f"authors-{year}.csv", r["Content-Disposition"]) + rows = list(csv.reader(io.StringIO(r.content.decode()))) + self.assertEqual(len(rows), 1) + self.assertIn("csvauthor@example.com", rows[0]) + + def test_download_submitters_csv(self): + self._member_login() + year = 2020 + SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 4, 1), + submitter_email="csvsubmitter@example.com", + ) + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url, {"download": "submitters"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "text/csv") + self.assertIn(f"submitters-{year}.csv", r["Content-Disposition"]) + rows = list(csv.reader(io.StringIO(r.content.decode()))) + self.assertEqual(len(rows), 1) + self.assertIn("csvsubmitter@example.com", rows[0]) diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index 01b8758c84..3bb107813a 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2016-2020, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,4 +15,5 @@ url(r"^meeting/(?P\d+)/(?Paffiliation|country)/$", views.meeting_stats), url(r"^meeting/(?:(?Paffiliation|country|total)/)?$", views.meetings_timeline), url(r"^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), + url(r"^annual_report_inputs/(?:(?P\d{4})/)?$", views.annual_report_inputs), ] diff --git a/ietf/stats/views.py b/ietf/stats/views.py index d61c9cab64..fe2fa82f55 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -1,8 +1,9 @@ -# Copyright The IETF Trust 2016-2020, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- import calendar +import csv import datetime import itertools import json @@ -12,7 +13,7 @@ from django.conf import settings from django.contrib.auth.decorators import login_required from django.core.cache import cache -from django.http import HttpResponseRedirect +from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import render, get_object_or_404 from django.urls import reverse as urlreverse from django.db.models import Count @@ -28,7 +29,7 @@ from ietf.person.models import Person from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName from ietf.meeting.models import Registration, Meeting -from ietf.ietfauth.utils import has_role +from ietf.ietfauth.utils import has_role, role_required from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, DEADLINE_TZINFO from ietf.meeting.helpers import get_current_ietf_meeting_num @@ -957,3 +958,46 @@ def time_key_fn(t): "possible_states": possible_states, "selected_state": selected_state, }) + + +@role_required("LLC Staff") +def annual_report_inputs(request, year=None): + if year is None and "year" in request.GET: + return HttpResponseRedirect( + urlreverse("ietf.stats.views.annual_report_inputs", kwargs={"year": request.GET["year"]}) + ) + year = int(year) if year else datetime.date.today().year - 1 + + from ietf.doc.models import NewRevisionDocEvent + from ietf.utils.reports import authors_by_year, submitters_by_year, unique_people + + download = request.GET.get("download") + if download in ("authors", "submitters"): + addresses = authors_by_year(year) if download == "authors" else submitters_by_year(year) + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="{download}-{year}.csv"' + writer = csv.writer(response) + writer.writerow(sorted(addresses)) + return response + + authors = authors_by_year(year) + submitters = submitters_by_year(year) + author_persons, author_nopersons = unique_people(authors) + submitter_persons, submitter_nopersons = unique_people(submitters) + + draft_count = len(set( + NewRevisionDocEvent.objects.filter( + doc__type_id="draft", time__year=year + ).values_list("doc__name", flat=True) + )) + + return render(request, "stats/annual_report_inputs.html", { + "year": year, + "author_count": len(authors), + "submitter_count": len(submitters), + "author_person_count": author_persons.count(), + "author_noperson_count": len(author_nopersons), + "submitter_person_count": submitter_persons.count(), + "submitter_noperson_count": len(submitter_nopersons), + "draft_count": draft_count, + }) diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html index 43ca025e28..b6b744ec21 100644 --- a/ietf/templates/base/menu.html +++ b/ietf/templates/base/menu.html @@ -1,4 +1,4 @@ -{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin %} {% origin %} {% load ietf_filters managed_groups wg_menu active_groups_menu group_filters cache meetings_filters %} @@ -453,6 +453,14 @@ {% endif %} + {% if user|has_role:"LLC Staff" %} +
  • + + Annual report inputs + +
    • + {% endif %}
  • diff --git a/ietf/templates/stats/annual_report_inputs.html b/ietf/templates/stats/annual_report_inputs.html new file mode 100644 index 0000000000..15add7ece3 --- /dev/null +++ b/ietf/templates/stats/annual_report_inputs.html @@ -0,0 +1,55 @@ +{# Copyright The IETF Trust 2026, All Rights Reserved #} +{% extends "base.html" %} +{% load origin %} +{% load ietf_filters static %} +{% block content %} + {% origin %} +

    {% block title %}Annual Report Inputs for {{ year }}{% endblock %}

    +
    +
    + + + +
    +
    +

    Summary

    + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Email addressesUnique persons foundAddresses with no person recordApproximate unique people
    Authors{{ author_count }}{{ author_person_count }}{{ author_noperson_count }}{{ author_person_count|add:author_noperson_count }}
    Submitters{{ submitter_count }}{{ submitter_person_count }}{{ submitter_noperson_count }}{{ submitter_person_count|add:submitter_noperson_count }}
    +

    Drafts submitted in {{ year }}: {{ draft_count }}

    +

    Downloads

    + +{% endblock %} diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py index 9708f2e8c0..9a969a5217 100755 --- a/ietf/utils/reports.py +++ b/ietf/utils/reports.py @@ -1,6 +1,5 @@ -# Copyright The IETF Trust 2023-2024, All Rights Reserved +# Copyright The IETF Trust 2023-2026, All Rights Reserved -import csv from typing import List, Set, Tuple from django.db.models import QuerySet @@ -48,20 +47,3 @@ def unique_people(addresses: List[str]) -> Tuple["QuerySet[Person]", Set]: return (persons, set(addresses) - set(known_email)) -def write_reports(year: int) -> None: - authors = authors_by_year(year) - submitters = submitters_by_year(year) - print(f"authors: {len(authors)}") - print(f"submitters: {len(submitters)}") - persons, nopersons = unique_people(authors) - print(f"authors: unique persons: {len(persons)}, no person found: {len(nopersons)}") - persons, nopersons = unique_people(submitters) - print( - f"submitters: unique persons: {len(persons)}, no person found: {len(nopersons)}" - ) - with open("authors.csv", "w", newline="") as file: - writer = csv.writer(file) - writer.writerow(authors) - with open("submitters.csv", "w", newline="") as file: - writer = csv.writer(file) - writer.writerow(submitters) From ad9824961922cf68e48fcbd90e536a7ca01076d5 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 8 Jun 2026 13:14:20 -0500 Subject: [PATCH 10/10] Update ietf/utils/tests_reports.py Co-authored-by: Jennifer Richards --- ietf/utils/tests_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/utils/tests_reports.py b/ietf/utils/tests_reports.py index 4fa27eed70..83daa15cc1 100755 --- a/ietf/utils/tests_reports.py +++ b/ietf/utils/tests_reports.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2023, All Rights Reserved +# Copyright The IETF Trust 2026, All Rights Reserved import datetime