ci: merge main to release (ietf-tools#9320)

Author: rjsparks

dev/build/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250624T1543
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250807T1514
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive

dev/build/TARGET_BASE

@@ -1 +1 @@
-20250624T1543
+20250807T1514

docker/scripts/app-init-celery.sh

@@ -92,7 +92,14 @@ fi
 
 USER_BIN_PATH="/home/dev/.local/bin"
 WATCHMEDO="$USER_BIN_PATH/watchmedo"
-CELERY="$USER_BIN_PATH/celery"
+# Find a celery that works
+if [[ -x "$USER_BIN_PATH/celery" ]]; then
+    # This branch is used for dev
+    CELERY="$USER_BIN_PATH/celery"
+else
+    # This branch is used for sandbox instances
+    CELERY="/usr/local/bin/celery"
+fi
 trap 'trap "" TERM; cleanup' TERM
 # start celery in the background so we can trap the TERM signal
 if [[ -n "${DEV_MODE}" && -x "${WATCHMEDO}" ]]; then

ietf/api/__init__.py

@@ -145,5 +145,26 @@ def dehydrate(self, bundle, for_list=True):
 
 
 class Serializer(tastypie.serializers.Serializer):
+    OPTION_ESCAPE_NULLS = "datatracker-escape-nulls"
+
     def format_datetime(self, data):
         return data.astimezone(datetime.timezone.utc).replace(tzinfo=None).isoformat(timespec="seconds") + "Z"
+
+    def to_simple(self, data, options):
+        options = options or {}
+        simple_data = super().to_simple(data, options)
+        if (
+            options.get(self.OPTION_ESCAPE_NULLS, False) 
+            and isinstance(simple_data, str)
+        ):
+            # replace nulls with unicode "symbol for null character", \u2400
+            simple_data = simple_data.replace("\x00", "\u2400")
+        return simple_data
+
+    def to_etree(self, data, options=None, name=None, depth=0):
+        # lxml does not escape nulls on its own, so ask to_simple() to do it.
+        # This is mostly (only?) an issue when generating errors responses for
+        # fuzzers.
+        options = options or {}
+        options[self.OPTION_ESCAPE_NULLS] = True
+        return super().to_etree(data, options, name, depth)

ietf/api/tests.py

@@ -5,7 +5,7 @@
 import datetime
 import json
 import html
-import mock
+from unittest import mock
 import os
 import sys
 
@@ -41,6 +41,7 @@
 from ietf.utils.models import DumpInfo
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized, reload_db_objects
 
+from . import Serializer
 from .ietf_utils import is_valid_token, requires_api_token
 from .views import EmailIngestionError
 
@@ -1496,7 +1497,7 @@ def test_good_password(self):
         data = self.response_data(r)
         self.assertEqual(data["result"], "success")
 
-class TastypieApiTestCase(ResourceTestCaseMixin, TestCase):
+class TastypieApiTests(ResourceTestCaseMixin, TestCase):
     def __init__(self, *args, **kwargs):
         self.apps = {}
         for app_name in settings.INSTALLED_APPS:
@@ -1506,7 +1507,7 @@ def __init__(self, *args, **kwargs):
                 models_path = os.path.join(os.path.dirname(app.__file__), "models.py")
                 if os.path.exists(models_path):
                     self.apps[name] = app_name
-        super(TastypieApiTestCase, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
     def test_api_top_level(self):
         client = Client(Accept='application/json')
@@ -1541,6 +1542,21 @@ def test_all_model_resources_exist(self):
                     self.assertIn(model._meta.model_name, list(app_resources.keys()),
                         "There doesn't seem to be any API resource for model %s.models.%s"%(app.__name__,model.__name__,))
 
+    def test_serializer_to_etree_handles_nulls(self):
+        """Serializer to_etree() should handle a null character"""
+        serializer = Serializer()
+        try:
+            serializer.to_etree("string with no nulls in it")
+        except ValueError:
+            self.fail("serializer.to_etree raised ValueError on an ordinary string")
+        try:
+            serializer.to_etree("string with a \x00 in it")
+        except ValueError:
+            self.fail(
+                "serializer.to_etree raised ValueError on a string "
+                "containing a null character"
+            )
+
 
 class RfcdiffSupportTests(TestCase):
 

ietf/community/tests.py

@@ -1,7 +1,7 @@
 # Copyright The IETF Trust 2016-2023, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-import mock
+from unittest import mock
 from pyquery import PyQuery
 
 from django.test.utils import override_settings

ietf/doc/management/commands/find_github_backup_info.py

@@ -10,7 +10,7 @@
 from django.http import HttpRequest
 import lxml
 import bibtexparser
-import mock
+from unittest import mock
 import json
 import copy
 import random

ietf/doc/tests.py

@@ -3,7 +3,7 @@
 
 
 import datetime
-import mock
+from unittest import mock
 
 from pyquery import PyQuery
 

ietf/doc/tests_ballot.py

@@ -5,7 +5,7 @@
 import os
 import datetime
 import io
-import mock
+from unittest import mock
 
 from collections import Counter
 from pathlib import Path