From 4762e252552587bfc80c91244c9acb9cc59d820b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 4 Aug 2025 15:02:44 -0300 Subject: [PATCH 01/15] test: replace mock with unittest.mock (#9286) --- ietf/api/tests.py | 2 +- ietf/community/tests.py | 2 +- ietf/doc/tests.py | 2 +- ietf/doc/tests_ballot.py | 2 +- ietf/doc/tests_draft.py | 2 +- ietf/doc/tests_material.py | 2 +- ietf/doc/tests_review.py | 2 +- ietf/doc/tests_tasks.py | 2 +- ietf/group/tests.py | 2 +- ietf/group/tests_info.py | 2 +- ietf/idindex/tests.py | 2 +- ietf/ipr/management/tests.py | 2 +- ietf/ipr/tests.py | 2 +- ietf/meeting/tests_models.py | 2 +- ietf/meeting/tests_tasks.py | 2 +- ietf/meeting/tests_utils.py | 2 +- ietf/meeting/tests_views.py | 2 +- ietf/message/tests.py | 2 +- ietf/nomcom/management/tests.py | 2 +- ietf/nomcom/tests.py | 2 +- ietf/person/tests.py | 2 +- ietf/review/tests.py | 2 +- ietf/submit/tests.py | 2 +- ietf/sync/tests.py | 2 +- ietf/utils/management/tests.py | 2 +- ietf/utils/tests.py | 2 +- requirements.txt | 2 -- 27 files changed, 26 insertions(+), 28 deletions(-) diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 93a2195467..93515dd0cb 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -5,7 +5,7 @@ import datetime import json import html -import mock +from unittest import mock import os import sys diff --git a/ietf/community/tests.py b/ietf/community/tests.py index 1255ba46eb..04f1433d61 100644 --- a/ietf/community/tests.py +++ b/ietf/community/tests.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2016-2023, All Rights Reserved # -*- coding: utf-8 -*- -import mock +from unittest import mock from pyquery import PyQuery from django.test.utils import override_settings diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 1229df46c5..d3fba03bcc 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -10,7 +10,7 @@ from django.http import HttpRequest import lxml import bibtexparser -import mock +from unittest import mock import json import copy import random diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py index ec23f3d491..810ee598f6 100644 --- a/ietf/doc/tests_ballot.py +++ b/ietf/doc/tests_ballot.py @@ -3,7 +3,7 @@ import datetime -import mock +from unittest import mock from pyquery import PyQuery diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index 4753c4ff0c..576feb0582 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -5,7 +5,7 @@ import os import datetime import io -import mock +from unittest import mock from collections import Counter from pathlib import Path diff --git a/ietf/doc/tests_material.py b/ietf/doc/tests_material.py index c87341c95b..04779bdaf1 100644 --- a/ietf/doc/tests_material.py +++ b/ietf/doc/tests_material.py @@ -6,7 +6,7 @@ import shutil import io -from mock import call, patch +from unittest.mock import call, patch from pathlib import Path from pyquery import PyQuery diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py index 9850beca75..8c1fc99ffe 100644 --- a/ietf/doc/tests_review.py +++ b/ietf/doc/tests_review.py @@ -8,7 +8,7 @@ import os import shutil -from mock import patch, Mock +from unittest.mock import patch, Mock from requests import Response from django.apps import apps diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 8a6ffa8be1..29689cd596 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -2,7 +2,7 @@ import debug # pyflakes:ignore import datetime -import mock +from unittest import mock from pathlib import Path diff --git a/ietf/group/tests.py b/ietf/group/tests.py index 31f8cc45b5..229744388c 100644 --- a/ietf/group/tests.py +++ b/ietf/group/tests.py @@ -3,7 +3,7 @@ import datetime import json -import mock +from unittest import mock from django.urls import reverse as urlreverse from django.db.models import Q diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index aaf937ee43..eb85860ece 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -6,7 +6,7 @@ import datetime import io import bleach -import mock +from unittest import mock from unittest.mock import call, patch from pathlib import Path diff --git a/ietf/idindex/tests.py b/ietf/idindex/tests.py index 5cc7a7b3bb..ba6100550d 100644 --- a/ietf/idindex/tests.py +++ b/ietf/idindex/tests.py @@ -3,7 +3,7 @@ import datetime -import mock +from unittest import mock from pathlib import Path from tempfile import TemporaryDirectory diff --git a/ietf/ipr/management/tests.py b/ietf/ipr/management/tests.py index d84b0cfef4..d7acd65042 100644 --- a/ietf/ipr/management/tests.py +++ b/ietf/ipr/management/tests.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2021, All Rights Reserved # -*- coding: utf-8 -*- """Tests of ipr management commands""" -import mock +from unittest import mock import sys from django.core.management import call_command diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py index e0d00b5d1a..74fa540126 100644 --- a/ietf/ipr/tests.py +++ b/ietf/ipr/tests.py @@ -3,7 +3,7 @@ import datetime -import mock +from unittest import mock import re from pyquery import PyQuery diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py index e333ddad9a..869d9ec814 100644 --- a/ietf/meeting/tests_models.py +++ b/ietf/meeting/tests_models.py @@ -3,7 +3,7 @@ """Tests of models in the Meeting application""" import datetime -from mock import patch +from unittest.mock import patch from django.conf import settings from django.test import override_settings diff --git a/ietf/meeting/tests_tasks.py b/ietf/meeting/tests_tasks.py index 66de212899..0c442c4bf7 100644 --- a/ietf/meeting/tests_tasks.py +++ b/ietf/meeting/tests_tasks.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2025, All Rights Reserved import datetime -from mock import patch, call +from unittest.mock import patch, call from ietf.utils.test_utils import TestCase from ietf.utils.timezone import date_today from .factories import MeetingFactory diff --git a/ietf/meeting/tests_utils.py b/ietf/meeting/tests_utils.py index 391e017d68..7dd8f435e1 100644 --- a/ietf/meeting/tests_utils.py +++ b/ietf/meeting/tests_utils.py @@ -7,7 +7,7 @@ import json import jsonschema from json import JSONDecodeError -from mock import patch, Mock +from unittest.mock import patch, Mock from django.http import HttpResponse, JsonResponse from ietf.meeting.factories import MeetingFactory, RegistrationFactory, RegistrationTicketFactory diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 1aac2a6523..96a29c2297 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -12,7 +12,7 @@ import requests_mock from unittest import skipIf -from mock import call, patch, PropertyMock +from unittest.mock import call, patch, PropertyMock from pyquery import PyQuery from lxml.etree import tostring from io import StringIO, BytesIO diff --git a/ietf/message/tests.py b/ietf/message/tests.py index a677d5477e..e1bad9a1e6 100644 --- a/ietf/message/tests.py +++ b/ietf/message/tests.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2013-2020, All Rights Reserved # -*- coding: utf-8 -*- import datetime -import mock +from unittest import mock from smtplib import SMTPException diff --git a/ietf/nomcom/management/tests.py b/ietf/nomcom/management/tests.py index 7bda2b5aa5..08c0e1fe32 100644 --- a/ietf/nomcom/management/tests.py +++ b/ietf/nomcom/management/tests.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2021, All Rights Reserved # -*- coding: utf-8 -*- """Tests of nomcom management commands""" -import mock +from unittest import mock import sys from collections import namedtuple diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index ea17da6707..cc2e0826d3 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -4,7 +4,7 @@ import datetime import io -import mock +from unittest import mock import random import shutil diff --git a/ietf/person/tests.py b/ietf/person/tests.py index 61d9b0ed70..6326362fd8 100644 --- a/ietf/person/tests.py +++ b/ietf/person/tests.py @@ -4,7 +4,7 @@ import datetime import json -import mock +from unittest import mock from io import StringIO, BytesIO from PIL import Image diff --git a/ietf/review/tests.py b/ietf/review/tests.py index e9ddbd47af..5dc8f11e8e 100644 --- a/ietf/review/tests.py +++ b/ietf/review/tests.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2019-2020, All Rights Reserved # -*- coding: utf-8 -*- import datetime -import mock +from unittest import mock import debug # pyflakes:ignore from pyquery import PyQuery diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 7e70c55965..6b9002502b 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -5,7 +5,7 @@ import datetime import email import io -import mock +from unittest import mock import os import re import sys diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 14d65de0b2..182b6e24c4 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -6,7 +6,7 @@ import io import json import datetime -import mock +from unittest import mock import quopri import requests diff --git a/ietf/utils/management/tests.py b/ietf/utils/management/tests.py index d704999cd1..38be464c7f 100644 --- a/ietf/utils/management/tests.py +++ b/ietf/utils/management/tests.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2013-2020, All Rights Reserved # -*- coding: utf-8 -*- -import mock +from unittest import mock from django.core.management import call_command, CommandError from django.test import override_settings diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index ce1842236d..01433888fe 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -11,7 +11,7 @@ import shutil import types -from mock import call, patch +from unittest.mock import call, patch from pyquery import PyQuery from typing import Dict, List # pyflakes:ignore diff --git a/requirements.txt b/requirements.txt index 4eb573ce36..8ed354192a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,8 +51,6 @@ logging_tree>=1.9 # Used only by the showloggers management command lxml>=5.3.0 markdown>=3.3.6 types-markdown>=3.3.6 -mock>=4.0.3 # Used only by tests, of course -types-mock>=4.0.3 mypy~=1.7.0 # Version requirements determined by django-stubs. oic>=1.3 # Used only by tests Pillow>=9.1.0 From e0546b1543565c0a293d198db9b15f1dd5121600 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 4 Aug 2025 15:04:09 -0300 Subject: [PATCH 02/15] fix: blank=True for xml_version (#9285) --- .../0002_alter_submission_xml_version.py | 18 ++++++++++++++++++ ietf/submit/models.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 ietf/submit/migrations/0002_alter_submission_xml_version.py diff --git a/ietf/submit/migrations/0002_alter_submission_xml_version.py b/ietf/submit/migrations/0002_alter_submission_xml_version.py new file mode 100644 index 0000000000..275e6efd95 --- /dev/null +++ b/ietf/submit/migrations/0002_alter_submission_xml_version.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.23 on 2025-08-01 19:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("submit", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="submission", + name="xml_version", + field=models.CharField(blank=True, default=None, max_length=4, null=True), + ), + ] diff --git a/ietf/submit/models.py b/ietf/submit/models.py index 51f7541e31..1145f761b4 100644 --- a/ietf/submit/models.py +++ b/ietf/submit/models.py @@ -55,7 +55,7 @@ class Submission(models.Model): file_size = models.IntegerField(null=True, blank=True) document_date = models.DateField(null=True, blank=True) submission_date = models.DateField(default=date_today) - xml_version = models.CharField(null=True, max_length=4, default=None) + xml_version = models.CharField(null=True, blank=True, max_length=4, default=None) submitter = models.CharField(max_length=255, blank=True, help_text="Name and email of submitter, e.g. \"John Doe <john@example.org>\".") From 827f4e74a1b9a8e872634f31e9484a3dc8cd0842 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 4 Aug 2025 15:09:15 -0300 Subject: [PATCH 03/15] fix: escape nulls in XML api responses (#9283) * fix: escape nulls in XML api responses * refactor: use \u2400 instead of \0 Less likely to lead to null injection down the road * test: modern naming/python * test: test null char handling * test: remove unused vars --- ietf/api/__init__.py | 21 +++++++++++++++++++++ ietf/api/tests.py | 20 ++++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index 9fadab8e6f..d70866083e 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -145,5 +145,26 @@ def dehydrate(self, bundle, for_list=True): class Serializer(tastypie.serializers.Serializer): + OPTION_ESCAPE_NULLS = "datatracker-escape-nulls" + def format_datetime(self, data): return data.astimezone(datetime.timezone.utc).replace(tzinfo=None).isoformat(timespec="seconds") + "Z" + + def to_simple(self, data, options): + options = options or {} + simple_data = super().to_simple(data, options) + if ( + options.get(self.OPTION_ESCAPE_NULLS, False) + and isinstance(simple_data, str) + ): + # replace nulls with unicode "symbol for null character", \u2400 + simple_data = simple_data.replace("\x00", "\u2400") + return simple_data + + def to_etree(self, data, options=None, name=None, depth=0): + # lxml does not escape nulls on its own, so ask to_simple() to do it. + # This is mostly (only?) an issue when generating errors responses for + # fuzzers. + options = options or {} + options[self.OPTION_ESCAPE_NULLS] = True + return super().to_etree(data, options, name, depth) diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 93515dd0cb..865f877bfb 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -41,6 +41,7 @@ from ietf.utils.models import DumpInfo from ietf.utils.test_utils import TestCase, login_testing_unauthorized, reload_db_objects +from . import Serializer from .ietf_utils import is_valid_token, requires_api_token from .views import EmailIngestionError @@ -1496,7 +1497,7 @@ def test_good_password(self): data = self.response_data(r) self.assertEqual(data["result"], "success") -class TastypieApiTestCase(ResourceTestCaseMixin, TestCase): +class TastypieApiTests(ResourceTestCaseMixin, TestCase): def __init__(self, *args, **kwargs): self.apps = {} for app_name in settings.INSTALLED_APPS: @@ -1506,7 +1507,7 @@ def __init__(self, *args, **kwargs): models_path = os.path.join(os.path.dirname(app.__file__), "models.py") if os.path.exists(models_path): self.apps[name] = app_name - super(TastypieApiTestCase, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def test_api_top_level(self): client = Client(Accept='application/json') @@ -1541,6 +1542,21 @@ def test_all_model_resources_exist(self): self.assertIn(model._meta.model_name, list(app_resources.keys()), "There doesn't seem to be any API resource for model %s.models.%s"%(app.__name__,model.__name__,)) + def test_serializer_to_etree_handles_nulls(self): + """Serializer to_etree() should handle a null character""" + serializer = Serializer() + try: + serializer.to_etree("string with no nulls in it") + except ValueError: + self.fail("serializer.to_etree raised ValueError on an ordinary string") + try: + serializer.to_etree("string with a \x00 in it") + except ValueError: + self.fail( + "serializer.to_etree raised ValueError on a string " + "containing a null character" + ) + class RfcdiffSupportTests(TestCase): From 8c4bff875398fb7fc7de624c8155c8377276cec0 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 4 Aug 2025 15:11:12 -0300 Subject: [PATCH 04/15] test: guard against empty path (#9282) --- ietf/utils/test_runner.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index d0a5496283..a9b2e5d572 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -432,7 +432,11 @@ def get_template_paths(apps=None) -> list[str]: relative_path = pathlib.Path( dirpath.removeprefix(templatepath).lstrip("/") ) - if apps and relative_path.parts[0] not in apps: + if ( + apps + and len(relative_path.parts) > 0 + and relative_path.parts[0] not in apps + ): continue # skip uninteresting apps for filename in files: file_path = project_path / filename From 649de73cab6c3aed3e909280e80e02272bc362a8 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 4 Aug 2025 15:13:46 -0300 Subject: [PATCH 05/15] fix: validate review_requests_history params (#9281) * test: test null chars in GET params * fix: validate GET params --- ietf/group/tests_review.py | 39 +++++++++++++++++++++++++++++++++ ietf/group/views.py | 45 +++++++++++++++++++++++--------------- 2 files changed, 66 insertions(+), 18 deletions(-) diff --git a/ietf/group/tests_review.py b/ietf/group/tests_review.py index d671228953..89c755bb26 100644 --- a/ietf/group/tests_review.py +++ b/ietf/group/tests_review.py @@ -943,3 +943,42 @@ def test_requests_history_filter_page(self): self.assertNotContains(r, 'Assigned') self.assertNotContains(r, 'Accepted') self.assertNotContains(r, 'Completed') + + def test_requests_history_invalid_filter_parameters(self): + # First assignment as assigned + review_req = ReviewRequestFactory(state_id="assigned", doc=DocumentFactory()) + group = review_req.team + url = urlreverse( + "ietf.group.views.review_requests_history", + kwargs={"acronym": group.acronym}, + ) + invalid_reviewer_emails = [ + "%00null@example.com", # urlencoded null character + "null@exa%00mple.com", # urlencoded null character + "\x00null@example.com", # literal null character + "null@ex\x00ample.com", # literal null character + ] + for invalid_email in invalid_reviewer_emails: + r = self.client.get( + url + f"?reviewer_email={invalid_email}" + ) + self.assertEqual( + r.status_code, + 400, + f"should return a 400 response for reviewer_email={repr(invalid_email)}" + ) + + invalid_since_choices = [ + "forever", # not an option + "all\x00", # literal null character + "a%00ll", # urlencoded null character + ] + for invalid_since in invalid_since_choices: + r = self.client.get( + url + f"?since={invalid_since}" + ) + self.assertEqual( + r.status_code, + 400, + f"should return a 400 response for since={repr(invalid_since)}" + ) diff --git a/ietf/group/views.py b/ietf/group/views.py index bc79599722..3529b31f68 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -51,7 +51,13 @@ from django.contrib.auth.decorators import login_required from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery, TextField, Value from django.db.models.functions import Coalesce -from django.http import HttpResponse, HttpResponseRedirect, Http404, JsonResponse +from django.http import ( + HttpResponse, + HttpResponseRedirect, + Http404, + JsonResponse, + HttpResponseBadRequest, +) from django.shortcuts import render, redirect, get_object_or_404 from django.template.loader import render_to_string from django.urls import reverse as urlreverse @@ -96,11 +102,9 @@ from ietf.review.policies import get_reviewer_queue_policy from ietf.review.utils import (can_manage_review_requests_for_team, can_access_review_stats_for_team, - extract_revision_ordered_review_requests_for_documents_and_replaced, assign_review_request_to_reviewer, close_review_request, - suggested_review_requests_for_team, unavailable_periods_to_list, current_unavailable_periods_for_reviewers, @@ -686,13 +690,30 @@ def history(request, acronym, group_type=None): "can_add_comment": can_add_comment, })) + +class RequestsHistoryParamsForm(forms.Form): + SINCE_CHOICES = ( + (None, "1 month"), + ("3m", "3 months"), + ("6m", "6 months"), + ("1y", "1 year"), + ("2y", "2 years"), + ("all", "All"), + ) + + reviewer_email = forms.EmailField(required=False) + since = forms.ChoiceField(choices=SINCE_CHOICES, required=False) + def review_requests_history(request, acronym, group_type=None): group = get_group_or_404(acronym, group_type) if not group.features.has_reviews: raise Http404 - reviewer_email = request.GET.get("reviewer_email", None) + params = RequestsHistoryParamsForm(request.GET) + if not params.is_valid(): + return HttpResponseBadRequest("Invalid parameters") + reviewer_email = params.cleaned_data["reviewer_email"] or None if reviewer_email: history = ReviewAssignment.history.model.objects.filter( review_request__team__acronym=acronym, @@ -702,19 +723,7 @@ def review_requests_history(request, acronym, group_type=None): review_request__team__acronym=acronym) reviewer_email = '' - since_choices = [ - (None, "1 month"), - ("3m", "3 months"), - ("6m", "6 months"), - ("1y", "1 year"), - ("2y", "2 years"), - ("all", "All"), - ] - since = request.GET.get("since", None) - - if since not in [key for key, label in since_choices]: - since = None - + since = params.cleaned_data["since"] or None if since != "all": date_limit = { None: datetime.timedelta(days=31), @@ -731,7 +740,7 @@ def review_requests_history(request, acronym, group_type=None): "group": group, "acronym": acronym, "history": history, - "since_choices": since_choices, + "since_choices": params.SINCE_CHOICES, "since": since, "reviewer_email": reviewer_email })) From 39165a0b5d079459da601ea82f87c9981f6508b7 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 4 Aug 2025 15:25:54 -0300 Subject: [PATCH 06/15] fix: serve materials w/mixed-case exts (#9273) * fix: serve materials w/ mixed-case exts * fix: another endpoint+reorder regex --- ietf/meeting/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py index 5da24ddb6f..18b123b4d8 100644 --- a/ietf/meeting/urls.py +++ b/ietf/meeting/urls.py @@ -64,7 +64,7 @@ def get_redirect_url(self, *args, **kwargs): type_interim_patterns = [ url(r'^agenda/(?P[A-Za-z0-9-]+)-drafts.pdf$', views.session_draft_pdf), url(r'^agenda/(?P[A-Za-z0-9-]+)-drafts.tgz$', views.session_draft_tarfile), - url(r'^materials/%(document)s(?P\.[a-z0-9]+)$' % settings.URL_REGEXPS, views.materials_document), + url(r'^materials/%(document)s(?P\.[A-Za-z0-9]+)$' % settings.URL_REGEXPS, views.materials_document), url(r'^materials/%(document)s/?$' % settings.URL_REGEXPS, views.materials_document), url(r'^agenda.json$', views.agenda_json) ] @@ -85,7 +85,7 @@ def get_redirect_url(self, *args, **kwargs): url(r'^week-view(?:.html)?/?$', AgendaRedirectView.as_view(pattern_name='agenda', permanent=True)), url(r'^materials(?:.html)?/?$', views.materials), url(r'^request_minutes/?$', views.request_minutes), - url(r'^materials/%(document)s(?P\.[a-z0-9]+)?/?$' % settings.URL_REGEXPS, views.materials_document), + url(r'^materials/%(document)s(?P\.[A-Za-z0-9]+)?/?$' % settings.URL_REGEXPS, views.materials_document), url(r'^session/?$', views.materials_editable_groups), url(r'^proceedings(?:.html)?/?$', views.proceedings), url(r'^proceedings(?:.html)?/finalize/?$', views.finalize_proceedings), From afb0d2d245a11384d73e8f0cc0d31150dd91f80a Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 4 Aug 2025 17:16:47 -0300 Subject: [PATCH 07/15] chore(deps): pin jsonfield version (#9267) At least nominally, 3.2.0 requires py3.10. Package is deprecated. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8ed354192a..1b00cf81a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,7 +44,7 @@ hashids>=1.3.1 html2text>=2020.1.16 # Used only to clean comment field of secr/sreq html5lib>=1.1 # Only used in tests inflect>= 6.0.2 -jsonfield>=3.1.0 # for SubmissionCheck. This is https://github.com/bradjasper/django-jsonfield/. +jsonfield>=3.1.0,<3.2.0 # 3.2.0 needs py3.10; deprecated-replace with Django JSONField jsonschema[format]>=4.2.1 jwcrypto>=1.2 # for signed notifications - this is aspirational, and is not really used. logging_tree>=1.9 # Used only by the showloggers management command From 6494ce880631ce798424a3a57db88f50f6ebf370 Mon Sep 17 00:00:00 2001 From: rjsparks <10996692+rjsparks@users.noreply.github.com> Date: Mon, 4 Aug 2025 20:29:21 +0000 Subject: [PATCH 08/15] ci: update base image target version to 20250804T2017 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 8317195446..662aee950f 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20250624T1543 +FROM ghcr.io/ietf-tools/datatracker-app-base:20250804T2017 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 1f2e39a0a2..d2c3dd6fc9 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20250624T1543 +20250804T2017 From e220bc89b464aab54e11c6698f074ed51982c715 Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Mon, 4 Aug 2025 22:43:44 +0200 Subject: [PATCH 09/15] Add link to reviewers's reviews (#9272) --- ietf/templates/person/profile.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ietf/templates/person/profile.html b/ietf/templates/person/profile.html index 1424f037a1..a78a90412f 100644 --- a/ietf/templates/person/profile.html +++ b/ietf/templates/person/profile.html @@ -50,7 +50,11 @@

Roles

{% for role in person.role_set.all|active_roles %} - {{ role.name.name }} + {{ role.name.name }} + {% if role.name.name == 'Reviewer' %} + (See reviews) + {% endif %} + {{ role.group.name }} ({{ role.group.acronym }}) From e3b87d9459c597731d828482bd13041931fe0a2d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 4 Aug 2025 22:28:31 -0300 Subject: [PATCH 10/15] chore: different celery path for sandboxes (#9300) * chore: different celery path for sandboxes * chore: typo --- docker/scripts/app-init-celery.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/scripts/app-init-celery.sh b/docker/scripts/app-init-celery.sh index 5788b943da..17925633d2 100755 --- a/docker/scripts/app-init-celery.sh +++ b/docker/scripts/app-init-celery.sh @@ -92,7 +92,14 @@ fi USER_BIN_PATH="/home/dev/.local/bin" WATCHMEDO="$USER_BIN_PATH/watchmedo" -CELERY="$USER_BIN_PATH/celery" +# Find a celery that works +if [[ -x "$USER_BIN_PATH/celery" ]]; then + # This branch is used for dev + CELERY="$USER_BIN_PATH/celery" +else + # This branch is used for sandbox instances + CELERY="/usr/local/bin/celery" +fi trap 'trap "" TERM; cleanup' TERM # start celery in the background so we can trap the TERM signal if [[ -n "${DEV_MODE}" && -x "${WATCHMEDO}" ]]; then From b8e135b928f9d67c83e6ef6fda6c273fdb106748 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 5 Aug 2025 14:38:30 -0300 Subject: [PATCH 11/15] chore: remove find_github_backup_info.py (#9307) --- .../commands/find_github_backup_info.py | 74 ------------------- requirements.txt | 1 - 2 files changed, 75 deletions(-) delete mode 100644 ietf/doc/management/commands/find_github_backup_info.py diff --git a/ietf/doc/management/commands/find_github_backup_info.py b/ietf/doc/management/commands/find_github_backup_info.py deleted file mode 100644 index f1f71452df..0000000000 --- a/ietf/doc/management/commands/find_github_backup_info.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright The IETF Trust 2020, All Rights Reserved - - -import github3 - -from collections import Counter -from urllib.parse import urlparse - -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError - -from ietf.doc.models import DocExtResource -from ietf.group.models import GroupExtResource -from ietf.person.models import PersonExtResource - -# TODO: Think more about submodules. This currently will only take top level repos, with the assumption that the clone will include arguments to grab all the submodules. -# As a consequence, we might end up pulling more than we need (or that the org or user expected) -# Make sure this is what we want. - -class Command(BaseCommand): - help = ('Locate information about github repositories to backup') - - def add_arguments(self, parser): - parser.add_argument('--verbose', dest='verbose', action='store_true', help='Show counts of types of repositories') - - def handle(self, *args, **options): - - if not (hasattr(settings,'GITHUB_BACKUP_API_KEY') and settings.GITHUB_BACKUP_API_KEY): - raise CommandError("ERROR: can't find GITHUB_BACKUP_API_KEY") # TODO: at >= py3.1, use returncode - - github = github3.login(token = settings.GITHUB_BACKUP_API_KEY) - owners = dict() - repos = set() - - for cls in (DocExtResource, GroupExtResource, PersonExtResource): - for res in cls.objects.filter(name_id__in=('github_repo','github_org')): - path_parts = urlparse(res.value).path.strip('/').split('/') - if not path_parts or not path_parts[0]: - continue - - owner = path_parts[0] - - if owner not in owners: - try: - gh_owner = github.user(username=owner) - owners[owner] = gh_owner - except github3.exceptions.NotFoundError: - continue - - if gh_owner.type in ('User', 'Organization'): - if len(path_parts) > 1: - repo = path_parts[1] - if (owner, repo) not in repos: - try: - github.repository(owner,repo) - repos.add( (owner, repo) ) - except github3.exceptions.NotFoundError: - continue - else: - for repo in github.repositories_by(owner): - repos.add( (owner, repo.name) ) - - owner_types = Counter([owners[owner].type for owner in owners]) - if options['verbose']: - self.stdout.write("Owners:") - for key in owner_types: - self.stdout.write(" %s: %s"%(key,owner_types[key])) - self.stdout.write("Repositories: %d" % len(repos)) - for repo in sorted(repos): - self.stdout.write(" https://github.com/%s/%s" % repo ) - else: - for repo in sorted(repos): - self.stdout.write("%s/%s" % repo ) - diff --git a/requirements.txt b/requirements.txt index 1b00cf81a2..f82bfc4101 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,6 @@ drf-spectacular>=0.27 drf-standardized-errors[openapi] >= 0.14 types-docutils>=0.18.1 factory-boy>=3.3 -github3.py>=3.2.0 gunicorn>=20.1.0 hashids>=1.3.1 html2text>=2020.1.16 # Used only to clean comment field of secr/sreq From beb873efc8a98cc5fe144304ebc050faeb814371 Mon Sep 17 00:00:00 2001 From: rjsparks <10996692+rjsparks@users.noreply.github.com> Date: Tue, 5 Aug 2025 17:51:24 +0000 Subject: [PATCH 12/15] ci: update base image target version to 20250805T1738 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 662aee950f..3d5520babe 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20250804T2017 +FROM ghcr.io/ietf-tools/datatracker-app-base:20250805T1738 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index d2c3dd6fc9..90d83abf03 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20250804T2017 +20250805T1738 From ebe6fbf046590c9b6f08560075b760f2164f1f2a Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 7 Aug 2025 12:13:41 -0300 Subject: [PATCH 13/15] feat: personless User deletion via admin (#9312) * feat: admin to allow user deletion * fix: permissions + drop dangerous action * chore: minor style lint * fix: avoid limit on a queryset delete * feat: User age filter * feat: show useful fields on User admin * chore: fix lint * fix: reverse direction of age filter --- ietf/ietfauth/admin.py | 136 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 ietf/ietfauth/admin.py diff --git a/ietf/ietfauth/admin.py b/ietf/ietfauth/admin.py new file mode 100644 index 0000000000..c2914f9efa --- /dev/null +++ b/ietf/ietfauth/admin.py @@ -0,0 +1,136 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +import datetime + +from django.conf import settings +from django.contrib import admin, messages +from django.contrib.admin import action +from django.contrib.admin.actions import delete_selected as default_delete_selected +from django.contrib.auth.admin import UserAdmin +from django.contrib.auth.models import User +from django.utils import timezone + + +# Replace default UserAdmin with our custom one +admin.site.unregister(User) + + +class AgeListFilter(admin.SimpleListFilter): + title = "account age" + parameter_name = "age" + + def lookups(self, request, model_admin): + return [ + ("1day", "> 1 day"), + ("3days", "> 3 days"), + ("1week", "> 1 week"), + ("1month", "> 1 month"), + ("1year", "> 1 year"), + ] + + def queryset(self, request, queryset): + deltas = { + "1day": datetime.timedelta(days=1), + "3days": datetime.timedelta(days=3), + "1week": datetime.timedelta(weeks=1), + "1month": datetime.timedelta(days=30), + "1year": datetime.timedelta(days=365), + } + if self.value(): + return queryset.filter(date_joined__lt=timezone.now()-deltas[self.value()]) + return queryset + + +@admin.register(User) +class CustomUserAdmin(UserAdmin): + list_display = ( + "username", + "person", + "date_joined", + "last_login", + "is_staff", + ) + list_filter = list(UserAdmin.list_filter) + [ + AgeListFilter, + ("person", admin.EmptyFieldListFilter), + ] + actions = ["delete_selected"] + + @action( + permissions=["delete"], description="Delete personless %(verbose_name_plural)s" + ) + def delete_selected(self, request, queryset): + """Delete selected action restricted to Users with a null Person field + + This displaces the default delete_selected action with a safer one that will + only delete personless Users. It is done this way instead of by introducing + a new action so that we can simply hand off to the default action (imported + as default_delete_selected()) without having to adjust its template (and maybe + other things) to make it work with a different action name. + """ + already_confirmed = bool(request.POST.get("post")) + personless_queryset = queryset.filter(person__isnull=True) + original_count = queryset.count() + personless_count = personless_queryset.count() + if personless_count > original_count: + # Refuse to act if the count increased! + self.message_user( + request, + ( + "Limiting the selection to Users without a Person INCREASED the " + "count from {} to {}. This should not happen and probably means a " + "concurrent change to the database affected this request. Please " + "try again.".format(original_count, personless_count) + ), + level=messages.ERROR, + ) + return None # return to changelist + + # Display warning/info if this is showing the confirmation page + if not already_confirmed: + if personless_count < original_count: + self.message_user( + request, + ( + "Limiting the selection to Users without a Person reduced the " + "count from {} to {}. Only {} will be deleted.".format( + original_count, personless_count, personless_count + ) + ), + level=messages.WARNING, + ) + else: + self.message_user( + request, + "Confirmed that all selected Users had no Persons.", + ) + + # Django limits the number of fields in a request. The delete form itself + # includes a few metadata fields, so give it a little padding. The default + # limit is 1000 and everything will break if it's a small number, so not + # bothering to check that it's > 10. + max_count = settings.DATA_UPLOAD_MAX_NUMBER_FIELDS - 10 + if personless_count > max_count: + self.message_user( + request, + ( + f"Only {max_count} Users can be deleted at once. Will only delete " + f"the first {max_count} selected Personless Users." + ), + level=messages.WARNING, + ) + # delete() doesn't like a queryset limited via [:max_count], so do an + # equivalent filter. + last_to_delete = personless_queryset.order_by("pk")[max_count] + personless_queryset = personless_queryset.filter(pk__lt=last_to_delete.pk) + + if already_confirmed and personless_count != original_count: + # After confirmation, none of the above filtering should change anything. + # Refuse to delete if the DB moved underneath us. + self.message_user( + request, + "Queryset count changed, nothing deleted. Please try again.", + level=messages.ERROR, + ) + return None + + return default_delete_selected(self, request, personless_queryset) From 86bce86731048f2dde04ace47af6425b775e23e9 Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Thu, 7 Aug 2025 11:14:20 -0400 Subject: [PATCH 14/15] feat: use icalendar instead manual template (#9187) * feat: use icalendar instead manual template * avoid code duplication * code cleanup * ruff ruff * remove comments * add custom field with meeting's local Time zone * more code cleanup * remove unused template for ical * pyflakes: remove unused imports and vars * improve tests and code coverage * remove commented line * change URL in ical to use session material page --- ietf/meeting/tests_views.py | 53 +++++++---- ietf/meeting/views.py | 148 +++++++++++++++++++++++++++--- ietf/templates/meeting/agenda.ics | 32 ------- requirements.txt | 1 + 4 files changed, 168 insertions(+), 66 deletions(-) delete mode 100644 ietf/templates/meeting/agenda.ics diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 96a29c2297..f382772485 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -15,6 +15,7 @@ from unittest.mock import call, patch, PropertyMock from pyquery import PyQuery from lxml.etree import tostring +from icalendar import Calendar from io import StringIO, BytesIO from bs4 import BeautifulSoup from urllib.parse import urlparse, urlsplit @@ -384,9 +385,6 @@ def test_meeting_agenda(self): r = self.client.get(ical_url) assert_ical_response_is_valid(self, r) - self.assertContains(r, "BEGIN:VTIMEZONE") - self.assertContains(r, "END:VTIMEZONE") - self.assertContains(r, meeting.time_zone, msg_prefix="time_zone should appear in its original case") self.assertNotEqual( meeting.time_zone, meeting.time_zone.lower(), @@ -405,21 +403,32 @@ def test_meeting_agenda(self): assert_ical_response_is_valid(self, r) self.assertContains(r, session.group.acronym) self.assertContains(r, session.group.name) - self.assertContains(r, session.remote_instructions) - self.assertContains(r, slot.location.name) - self.assertContains(r, 'https://onsite.example.com') - self.assertContains(r, 'https://meetecho.example.com') - self.assertContains(r, "BEGIN:VTIMEZONE") - self.assertContains(r, "END:VTIMEZONE") - self.assertContains(r, session.agenda().get_href()) - self.assertContains( - r, + cal = Calendar.from_ical(r.content) + events = [component for component in cal.walk() if component.name == "VEVENT"] + + self.assertEqual(len(events), 2) + self.assertIn(session.remote_instructions, events[0].get('description')) + self.assertIn("Onsite tool: https://onsite.example.com", events[0].get('description')) + self.assertIn("Meetecho: https://meetecho.example.com", events[0].get('description')) + self.assertIn(f"Agenda {session.agenda().get_href()}", events[0].get('description')) + session_materials_url = settings.IDTRACKER_BASE_URL + urlreverse( + 'ietf.meeting.views.session_details', + kwargs=dict(num=meeting.number, acronym=session.group.acronym) + ) + self.assertIn(f"Session materials: {session_materials_url}", events[0].get('description')) + self.assertIn( urlreverse( 'ietf.meeting.views.session_details', kwargs=dict(num=meeting.number, acronym=session.group.acronym)), - msg_prefix='ical should contain link to meeting materials page for session') + events[0].get('description')) + self.assertEqual( + session_materials_url, + events[0].get('url') + ) + self.assertContains(r, f"LOCATION:{slot.location.name}") + # Floor Plan r = self.client.get(urlreverse('floor-plan', kwargs=dict(num=meeting.number))) self.assertEqual(r.status_code, 200) @@ -1049,32 +1058,36 @@ def test_group_ical(self): s1 = Session.objects.filter(meeting=meeting, group__acronym="mars").first() a1 = s1.official_timeslotassignment() t1 = a1.timeslot + # Create an extra session t2 = TimeSlotFactory.create( meeting=meeting, - time=meeting.tz().localize( + time=pytz.utc.localize( datetime.datetime.combine(meeting.date, datetime.time(11, 30)) ) ) + s2 = SessionFactory.create(meeting=meeting, group=s1.group, add_to_schedule=False) SchedTimeSessAssignment.objects.create(timeslot=t2, session=s2, schedule=meeting.schedule) - # + url = urlreverse('ietf.meeting.views.agenda_ical', kwargs={'num':meeting.number, 'acronym':s1.group.acronym, }) r = self.client.get(url) assert_ical_response_is_valid(self, r, expected_event_summaries=['mars - Martian Special Interest Group'], expected_event_count=2) - self.assertContains(r, t1.local_start_time().strftime('%Y%m%dT%H%M%S')) - self.assertContains(r, t2.local_start_time().strftime('%Y%m%dT%H%M%S')) - # + self.assertContains(r, f"DTSTART:{t1.time.strftime('%Y%m%dT%H%M%SZ')}") + self.assertContains(r, f"DTEND:{(t1.time + t1.duration).strftime('%Y%m%dT%H%M%SZ')}") + self.assertContains(r, f"DTSTART:{t2.time.strftime('%Y%m%dT%H%M%SZ')}") + self.assertContains(r, f"DTEND:{(t2.time + t2.duration).strftime('%Y%m%dT%H%M%SZ')}") + url = urlreverse('ietf.meeting.views.agenda_ical', kwargs={'num':meeting.number, 'session_id':s1.id, }) r = self.client.get(url) assert_ical_response_is_valid(self, r, expected_event_summaries=['mars - Martian Special Interest Group'], expected_event_count=1) - self.assertContains(r, t1.local_start_time().strftime('%Y%m%dT%H%M%S')) - self.assertNotContains(r, t2.local_start_time().strftime('%Y%m%dT%H%M%S')) + self.assertContains(r, f"DTSTART:{t1.time.strftime('%Y%m%dT%H%M%SZ')}") + self.assertNotContains(r, f"DTSTART:{t2.time.strftime('%Y%m%dT%H%M%SZ')}") def test_parse_agenda_filter_params(self): def _r(show=(), hide=(), showtypes=(), hidetypes=()): diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 85eda5a8f4..7fa3d21259 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -118,6 +118,9 @@ UploadAgendaForm, UploadBlueSheetForm, UploadMinutesForm, UploadSlidesForm, UploadNarrativeMinutesForm) +from icalendar import Calendar, Event +from ietf.doc.templatetags.ietf_filters import absurl + request_summary_exclude_group_types = ['team'] @@ -137,6 +140,10 @@ def send_interim_change_notice(request, meeting): message.related_groups.add(group) send_mail_message(request, message) +def parse_ical_line_endings(ical): + """Parse icalendar line endings to ensure they are RFC 5545 compliant""" + return re.sub(r'\r(?!\n)|(?=20.1.0 hashids>=1.3.1 html2text>=2020.1.16 # Used only to clean comment field of secr/sreq html5lib>=1.1 # Only used in tests +icalendar>=5.0.0 inflect>= 6.0.2 jsonfield>=3.1.0,<3.2.0 # 3.2.0 needs py3.10; deprecated-replace with Django JSONField jsonschema[format]>=4.2.1 From 666e9c53b45bc2eeda80ccbea66abeaa01df9830 Mon Sep 17 00:00:00 2001 From: rjsparks <10996692+rjsparks@users.noreply.github.com> Date: Thu, 7 Aug 2025 15:27:02 +0000 Subject: [PATCH 15/15] ci: update base image target version to 20250807T1514 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 3d5520babe..0ecf9566ef 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20250805T1738 +FROM ghcr.io/ietf-tools/datatracker-app-base:20250807T1514 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 90d83abf03..327fb48da4 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20250805T1738 +20250807T1514