ci: merge main to release

dev/build/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250624T1543
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250807T1514
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive

dev/build/TARGET_BASE

@@ -1 +1 @@
-20250624T1543
+20250807T1514

docker/scripts/app-init-celery.sh

@@ -92,7 +92,14 @@ fi
 
 USER_BIN_PATH="/home/dev/.local/bin"
 WATCHMEDO="$USER_BIN_PATH/watchmedo"
-CELERY="$USER_BIN_PATH/celery"
+# Find a celery that works
+if [[ -x "$USER_BIN_PATH/celery" ]]; then
+    # This branch is used for dev
+    CELERY="$USER_BIN_PATH/celery"
+else
+    # This branch is used for sandbox instances
+    CELERY="/usr/local/bin/celery"
+fi
 trap 'trap "" TERM; cleanup' TERM
 # start celery in the background so we can trap the TERM signal
 if [[ -n "${DEV_MODE}" && -x "${WATCHMEDO}" ]]; then

ietf/api/__init__.py

@@ -145,5 +145,26 @@ def dehydrate(self, bundle, for_list=True):
 
 
 class Serializer(tastypie.serializers.Serializer):
+    OPTION_ESCAPE_NULLS = "datatracker-escape-nulls"
+
     def format_datetime(self, data):
         return data.astimezone(datetime.timezone.utc).replace(tzinfo=None).isoformat(timespec="seconds") + "Z"
+
+    def to_simple(self, data, options):
+        options = options or {}
+        simple_data = super().to_simple(data, options)
+        if (
+            options.get(self.OPTION_ESCAPE_NULLS, False) 
+            and isinstance(simple_data, str)
+        ):
+            # replace nulls with unicode "symbol for null character", \u2400
+            simple_data = simple_data.replace("\x00", "\u2400")
+        return simple_data
+
+    def to_etree(self, data, options=None, name=None, depth=0):
+        # lxml does not escape nulls on its own, so ask to_simple() to do it.
+        # This is mostly (only?) an issue when generating errors responses for
+        # fuzzers.
+        options = options or {}
+        options[self.OPTION_ESCAPE_NULLS] = True
+        return super().to_etree(data, options, name, depth)

ietf/api/tests.py

@@ -5,7 +5,7 @@
 import datetime
 import json
 import html
-import mock
+from unittest import mock
 import os
 import sys
 
@@ -41,6 +41,7 @@
 from ietf.utils.models import DumpInfo
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized, reload_db_objects
 
+from . import Serializer
 from .ietf_utils import is_valid_token, requires_api_token
 from .views import EmailIngestionError
 
@@ -1496,7 +1497,7 @@ def test_good_password(self):
         data = self.response_data(r)
         self.assertEqual(data["result"], "success")
 
-class TastypieApiTestCase(ResourceTestCaseMixin, TestCase):
+class TastypieApiTests(ResourceTestCaseMixin, TestCase):
     def __init__(self, *args, **kwargs):
         self.apps = {}
         for app_name in settings.INSTALLED_APPS:
@@ -1506,7 +1507,7 @@ def __init__(self, *args, **kwargs):
                 models_path = os.path.join(os.path.dirname(app.__file__), "models.py")
                 if os.path.exists(models_path):
                     self.apps[name] = app_name
-        super(TastypieApiTestCase, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
     def test_api_top_level(self):
         client = Client(Accept='application/json')
@@ -1541,6 +1542,21 @@ def test_all_model_resources_exist(self):
                     self.assertIn(model._meta.model_name, list(app_resources.keys()),
                         "There doesn't seem to be any API resource for model %s.models.%s"%(app.__name__,model.__name__,))
 
+    def test_serializer_to_etree_handles_nulls(self):
+        """Serializer to_etree() should handle a null character"""
+        serializer = Serializer()
+        try:
+            serializer.to_etree("string with no nulls in it")
+        except ValueError:
+            self.fail("serializer.to_etree raised ValueError on an ordinary string")
+        try:
+            serializer.to_etree("string with a \x00 in it")
+        except ValueError:
+            self.fail(
+                "serializer.to_etree raised ValueError on a string "
+                "containing a null character"
+            )
+
 
 class RfcdiffSupportTests(TestCase):
 

ietf/community/tests.py

@@ -1,7 +1,7 @@
 # Copyright The IETF Trust 2016-2023, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-import mock
+from unittest import mock
 from pyquery import PyQuery
 
 from django.test.utils import override_settings

ietf/doc/tests.py

@@ -10,7 +10,7 @@
 from django.http import HttpRequest
 import lxml
 import bibtexparser
-import mock
+from unittest import mock
 import json
 import copy
 import random

ietf/doc/tests_ballot.py

@@ -3,7 +3,7 @@
 
 
 import datetime
-import mock
+from unittest import mock
 
 from pyquery import PyQuery
 

ietf/doc/tests_draft.py

@@ -5,7 +5,7 @@
 import os
 import datetime
 import io
-import mock
+from unittest import mock
 
 from collections import Counter
 from pathlib import Path

ietf/doc/tests_material.py

@@ -6,7 +6,7 @@
 import shutil
 import io
 
-from mock import call, patch
+from unittest.mock import call, patch
 from pathlib import Path
 from pyquery import PyQuery
 

ietf/doc/tests_review.py

@@ -8,7 +8,7 @@
 import os
 import shutil
 
-from mock import patch, Mock
+from unittest.mock import patch, Mock
 from requests import Response
 
 from django.apps import apps

ietf/doc/tests_tasks.py

@@ -2,7 +2,7 @@
 
 import debug    # pyflakes:ignore
 import datetime
-import mock
+from unittest import mock
 
 from pathlib import Path
 

ietf/group/tests.py

@@ -3,7 +3,7 @@
 
 import datetime
 import json
-import mock
+from unittest import mock
 
 from django.urls import reverse as urlreverse
 from django.db.models import Q

ietf/group/tests_info.py

@@ -6,7 +6,7 @@
 import datetime
 import io
 import bleach
-import mock
+from unittest import mock
 
 from unittest.mock import call, patch
 from pathlib import Path

ietf/group/tests_review.py

@@ -943,3 +943,42 @@ def test_requests_history_filter_page(self):
         self.assertNotContains(r, 'Assigned')
         self.assertNotContains(r, 'Accepted')
         self.assertNotContains(r, 'Completed')
+
+    def test_requests_history_invalid_filter_parameters(self):
+        # First assignment as assigned
+        review_req = ReviewRequestFactory(state_id="assigned", doc=DocumentFactory())
+        group = review_req.team
+        url = urlreverse(
+            "ietf.group.views.review_requests_history",
+            kwargs={"acronym": group.acronym},
+        )
+        invalid_reviewer_emails = [
+            "%00null@example.com",  # urlencoded null character
+            "null@exa%00mple.com",  # urlencoded null character
+            "\x00null@example.com",  # literal null character
+            "null@ex\x00ample.com",  # literal null character
+        ]
+        for invalid_email in invalid_reviewer_emails:
+            r = self.client.get(
+                url + f"?reviewer_email={invalid_email}"
+            )
+            self.assertEqual(
+                r.status_code, 
+                400,
+                f"should return a 400 response for reviewer_email={repr(invalid_email)}"
+            )
+
+        invalid_since_choices = [
+            "forever",  # not an option
+            "all\x00",  # literal null character
+            "a%00ll",  # urlencoded null character
+        ]
+        for invalid_since in invalid_since_choices:
+            r = self.client.get(
+                url + f"?since={invalid_since}"
+            )
+            self.assertEqual(
+                r.status_code, 
+                400,
+                f"should return a 400 response for since={repr(invalid_since)}"
+            )

ietf/group/views.py

@@ -51,7 +51,13 @@
 from django.contrib.auth.decorators import login_required
 from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery, TextField, Value
 from django.db.models.functions import Coalesce
-from django.http import HttpResponse, HttpResponseRedirect, Http404, JsonResponse
+from django.http import (
+    HttpResponse,
+    HttpResponseRedirect,
+    Http404,
+    JsonResponse,
+    HttpResponseBadRequest,
+)
 from django.shortcuts import render, redirect, get_object_or_404
 from django.template.loader import render_to_string
 from django.urls import reverse as urlreverse
@@ -96,11 +102,9 @@
 from ietf.review.policies import get_reviewer_queue_policy
 from ietf.review.utils import (can_manage_review_requests_for_team,
                                can_access_review_stats_for_team,
-
                                extract_revision_ordered_review_requests_for_documents_and_replaced,
                                assign_review_request_to_reviewer,
                                close_review_request,
-
                                suggested_review_requests_for_team,
                                unavailable_periods_to_list,
                                current_unavailable_periods_for_reviewers,
@@ -686,13 +690,30 @@ def history(request, acronym, group_type=None):
                       "can_add_comment": can_add_comment,
                   }))
 
+
+class RequestsHistoryParamsForm(forms.Form):
+    SINCE_CHOICES = (
+        (None, "1 month"),
+        ("3m", "3 months"),
+        ("6m", "6 months"),
+        ("1y", "1 year"),
+        ("2y", "2 years"),
+        ("all", "All"),
+    )
+
+    reviewer_email = forms.EmailField(required=False)
+    since = forms.ChoiceField(choices=SINCE_CHOICES, required=False)
+
 def review_requests_history(request, acronym, group_type=None):
     group = get_group_or_404(acronym, group_type)
     if not group.features.has_reviews:
         raise Http404
 
-    reviewer_email = request.GET.get("reviewer_email", None)
+    params = RequestsHistoryParamsForm(request.GET)
+    if not params.is_valid():
+        return HttpResponseBadRequest("Invalid parameters")
 
+    reviewer_email = params.cleaned_data["reviewer_email"] or None
     if reviewer_email:
         history = ReviewAssignment.history.model.objects.filter(
             review_request__team__acronym=acronym,
@@ -702,19 +723,7 @@ def review_requests_history(request, acronym, group_type=None):
             review_request__team__acronym=acronym)
         reviewer_email = ''
 
-    since_choices = [
-        (None, "1 month"),
-        ("3m", "3 months"),
-        ("6m", "6 months"),
-        ("1y", "1 year"),
-        ("2y", "2 years"),
-        ("all", "All"),
-    ]
-    since = request.GET.get("since", None)
-
-    if since not in [key for key, label in since_choices]:
-        since = None
-
+    since = params.cleaned_data["since"] or None
     if since != "all":
         date_limit = {
             None: datetime.timedelta(days=31),
@@ -731,7 +740,7 @@ def review_requests_history(request, acronym, group_type=None):
                       "group": group,
                       "acronym": acronym,
                       "history": history,
-                      "since_choices": since_choices,
+                      "since_choices": params.SINCE_CHOICES,
                       "since": since,
                       "reviewer_email": reviewer_email
                   }))