[pull] main from ietf-tools:main

dev/build/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20260604T1454
+FROM ghcr.io/ietf-tools/datatracker-app-base:20260605T2314
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive

dev/build/TARGET_BASE

@@ -1 +1 @@
-20260604T1454
+20260605T2314

dev/build/gunicorn.conf.py

@@ -7,6 +7,10 @@
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor
+from opentelemetry.instrumentation.pymemcache import PymemcacheInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
 
 # Bind all ipv4 interfaces (nginx uses loopback, but k8s health checks don't)
 _BIND_PORT = os.environ.get("DATATRACKER_GUNICORN_BIND_PORT", "8000")
@@ -167,15 +171,11 @@ def post_fork(server, worker):
         trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter))
 
         # Instrumentations
-        if "all" in enabled_telemetry or "django" in enabled_telemetry:
-            from opentelemetry.instrumentation.django import DjangoInstrumentor
+        if "all" in enabled_telemetry or "django" in enabled_telemetry: 
             DjangoInstrumentor().instrument()
-        if "all" in enabled_telemetry or "psycopg2" in enabled_telemetry:
-            from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor
+        if "all" in enabled_telemetry or "psycopg2" in enabled_telemetry: 
             Psycopg2Instrumentor().instrument()
-        if "all" in enabled_telemetry or "redis" in enabled_telemetry:
-            from opentelemetry.instrumentation.redis import RedisInstrumentor
-            RedisInstrumentor().instrument()
-        if "all" in enabled_telemetry or "requests" in enabled_telemetry:
-            from opentelemetry.instrumentation.requests import RequestsInstrumentor
+        if "all" in enabled_telemetry or "pymemcache" in enabled_telemetry: 
+            PymemcacheInstrumentor().instrument()
+        if "all" in enabled_telemetry or "requests" in enabled_telemetry: 
             RequestsInstrumentor().instrument()

docker-compose.yml

@@ -90,12 +90,6 @@ services:
             - .:/workspace
             - app-assets:/assets
 
-    redis:
-        image: redis:8
-        command: ['redis-server', '--save', '10', '1', '--loglevel', 'warning']
-        volumes:
-            - redis:/data
-
     replicator:
         build:
             context: .
@@ -175,4 +169,3 @@ volumes:
     app-assets:
     minio-data:
     blobdb-data:
-    redis:

docker/base.Dockerfile

@@ -59,10 +59,12 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends
 	libxtst6 \
 	libmagic-dev \
 	libmariadb-dev \
+	libmemcached-tools \
 	libyang2-tools \
 	locales \
 	make \
 	mariadb-client \
+	memcached \
 	nano \
 	netcat-traditional \
 	nodejs \

ietf/settings.py

@@ -1363,93 +1363,138 @@ def skip_unreadable_post(record):
     TEMPLATES[0]['OPTIONS']['context_processors'] += DEV_TEMPLATE_CONTEXT_PROCESSORS
 
 if "CACHES" not in locals():
-    # Would like to refuse to start when in prod mode, but this currently blocks
-    # the collectstatics call in the release GHA. We should probably arrange for that
-    # to have its own caches config, but in the meantime just let this fall back to
-    # the dev cache configuration.
-    #
-    # if SERVER_MODE == "production":
-    #     raise RuntimeError("Must set CACHES in settings_local for production mode")
-    CACHES = {
-        "default": {
-            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-            # "BACKEND": "django_redis.cache.RedisCache",
-            # "LOCATION": "redis://redis:6379/0",
-            # "OPTIONS": {
-            #     "CLIENT_CLASS": "ietf.utils.cache.SizeLimitingRedisClient",
-            # },
-            # "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
-            "VERSION": __version__,
-            "KEY_PREFIX": "ietf:dt",
-        },
-        "agenda": {
-            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-            # "BACKEND": "django_redis.cache.RedisCache",
-            # "LOCATION": "redis://redis:6379/0",
-            # "OPTIONS": {
-            #     "CLIENT_CLASS": "ietf.utils.cache.SizeLimitingRedisClient",
-            # },
-            # No release-specific VERSION setting.
-            "KEY_PREFIX": "ietf:dt:agenda",
-            # Key function is default except with sha384-encoded key
-            "KEY_FUNCTION": lambda key, key_prefix, version: (
-                f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
-            ),
-        },
-        "proceedings": {
-            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-            # "BACKEND": "django_redis.cache.RedisCache",
-            # "LOCATION": "redis://redis:6379/0",
-            # "OPTIONS": {
-            #     "CLIENT_CLASS": "ietf.utils.cache.SizeLimitingRedisClient",
-            # },
-            # No release-specific VERSION setting.
-            "KEY_PREFIX": "ietf:dt:proceedings",
-            # Key function is default except with sha384-encoded key
-            "KEY_FUNCTION": lambda key, key_prefix, version: (
-                f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
-            ),
-        },
-        "sessions": {
-            "BACKEND": "django_redis.cache.RedisCache",
-            "LOCATION": "redis://redis:6379/0",
-            "OPTIONS": {
-                "CLIENT_CLASS": "ietf.utils.cache.SizeLimitingRedisClient",
+    if SERVER_MODE == "production":
+        MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1")
+        MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211")
+        CACHES = {
+            "default": {
+                "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+                "VERSION": __version__,
+                "KEY_PREFIX": "ietf:dt",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
             },
-        },
-        "htmlized": {
-            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-            #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
-            "LOCATION": "/var/cache/datatracker/htmlized",
-            "OPTIONS": {
-                "MAX_ENTRIES": 1000,
+            "agenda": {
+                "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt:agenda",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
             },
-        },
-        "pdfized": {
-            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-            #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
-            "LOCATION": "/var/cache/datatracker/pdfized",
-            "OPTIONS": {
-                "MAX_ENTRIES": 1000,
+            "proceedings": {
+                "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt:proceedings",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
             },
-        },
-        "slowpages": {
-            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-            #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
-            "LOCATION": "/var/cache/datatracker/",
-            "OPTIONS": {
-                "MAX_ENTRIES": 5000,
+            "sessions": {
+                "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt",
             },
-        },
-        "celery-results": {
-            "BACKEND": "django_redis.cache.RedisCache",
-            "LOCATION": "redis://redis:6379/0",
-            "OPTIONS": {
-                "CLIENT_CLASS": "django_redis.client.DefaultClient",
+            "htmlized": {
+                "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
+                "LOCATION": "/a/cache/datatracker/htmlized",
+                "OPTIONS": {
+                    "MAX_ENTRIES": 100000,  # 100,000
+                },
             },
-            "KEY_PREFIX": "ietf:celery",
-        },
-    }
+            "pdfized": {
+                "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
+                "LOCATION": "/a/cache/datatracker/pdfized",
+                "OPTIONS": {
+                    "MAX_ENTRIES": 100000,  # 100,000
+                },
+            },
+            "slowpages": {
+                "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
+                "LOCATION": "/a/cache/datatracker/slowpages",
+                "OPTIONS": {
+                    "MAX_ENTRIES": 5000,
+                },
+            },
+            "celery-results": {
+                "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache",
+                "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+                "KEY_PREFIX": "ietf:celery",
+            },
+        }
+    else:
+        CACHES = {
+            "default": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                #'BACKEND': 'ietf.utils.cache.LenientMemcacheCache',
+                #'LOCATION': '127.0.0.1:11211',
+                #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+                "VERSION": __version__,
+                "KEY_PREFIX": "ietf:dt",
+            },
+            "agenda": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                # "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                # "LOCATION": "127.0.0.1:11211",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt:agenda",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
+            },
+            "proceedings": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                # "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                # "LOCATION": "127.0.0.1:11211",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt:proceedings",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
+            },
+            "sessions": {
+                "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+            },
+            "htmlized": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+                "LOCATION": "/var/cache/datatracker/htmlized",
+                "OPTIONS": {
+                    "MAX_ENTRIES": 1000,
+                },
+            },
+            "pdfized": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+                "LOCATION": "/var/cache/datatracker/pdfized",
+                "OPTIONS": {
+                    "MAX_ENTRIES": 1000,
+                },
+            },
+            "slowpages": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+                "LOCATION": "/var/cache/datatracker/",
+                "OPTIONS": {
+                    "MAX_ENTRIES": 5000,
+                },
+            },
+            "celery-results": {
+                "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache",
+                "LOCATION": "app:11211",
+                "KEY_PREFIX": "ietf:celery",
+            },
+        }
 
 PUBLISH_IPR_STATES = ['posted', 'removed', 'removed_objfalse']
 
@@ -1464,6 +1509,7 @@ def skip_unreadable_post(record):
     loaders = TEMPLATES[0]['OPTIONS']['loaders']
     loaders = tuple(l for e in loaders for l in (e[1] if isinstance(e, tuple) and "cached.Loader" in e[0] else (e,)))
     TEMPLATES[0]['OPTIONS']['loaders'] = loaders
+    SESSION_ENGINE = "django.contrib.sessions.backends.db"
 
     if 'SECRET_KEY' not in locals():
         SECRET_KEY = 'PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHka'

ietf/settings_test.py

@@ -69,9 +69,6 @@ def tempdir_with_cleanup(**kwargs):
 PHOTOS_DIR = os.path.join(MEDIA_ROOT, PHOTOS_DIRNAME)
 os.mkdir(PHOTOS_DIR)
 
-# Use database-backed sessions for tests
-SESSION_ENGINE = "django.contrib.sessions.backends.db"
-
 # Undo any developer-dependent middleware when running the tests
 MIDDLEWARE = [ c for c in MIDDLEWARE if not c in DEV_MIDDLEWARE ] # pyflakes:ignore
 

ietf/utils/cache.py

@@ -1,67 +1,20 @@
-# Copyright The IETF Trust 2026, All Rights Reserved
-from typing import Optional, Union, Any
+# Copyright The IETF Trust 2023, All Rights Reserved
+# -*- coding: utf-8 -*-
 
-from django.core.cache import BaseCache
 from django.core.cache.backends.base import DEFAULT_TIMEOUT
-from django_redis.client import DefaultClient, SentinelClient
-from redis import Redis
-from redis.typing import KeyT, EncodableT
+from django.core.cache.backends.memcached import PyMemcacheCache
+from pymemcache.exceptions import MemcacheServerError
 
-from ietf.utils.log import log
+from .log import log
 
 
-class EncodedValueTooBig(ValueError):
-    def __init__(self, *args, value_len):
-        super().__init__(*args)
-        self.value_len = value_len
-
-
-class SizeLimitingRedisClient(DefaultClient):
-    """Redis DefaultClient that refuses to cache large objects
-
-    Size applies to the literal cached value, which is _after_ serialization and
-    compression.
-
-    Set size limit with MAX_ENCODED_VALUE_LEN in the OPTIONS dict. Defaults to
-    1 MB.
-    """
-    def __init__(self, server, params: dict[str, Any], backend: BaseCache) -> None:
-        super().__init__(server, params, backend)
-        self.max_encoded_value_len = self._options.get("MAX_ENCODED_VALUE_LEN", 1 << 20)
-
-    def encode(self, value: EncodableT) -> Union[bytes, int]:
-        encoded = super().encode(value)
-        if isinstance(encoded, bytes) and len(encoded) > self.max_encoded_value_len:
-            raise EncodedValueTooBig(value_len=len(encoded))
-        return encoded
-
-    def set(
-        self,
-        key: KeyT,
-        value: EncodableT,
-        timeout: Optional[float] = DEFAULT_TIMEOUT,
-        version: Optional[int] = None,
-        client: Optional[Redis] = None,
-        nx: bool = False,
-        xx: bool = False,
-    ) -> bool:
+class LenientMemcacheCache(PyMemcacheCache):
+    """PyMemcacheCache backend that tolerates failed inserts due to object size"""
+    def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
         try:
-            return super().set(key, value, timeout, version, client, nx, xx)
-        except EncodedValueTooBig as err:
-            log(
-                f"Refused to cache large object for {key!r} "
-                f"({err.value_len} > {self.max_encoded_value_len} bytes)"
-            )
-            return False
-
-
-class SizeLimitingSentinelClient(SizeLimitingRedisClient, SentinelClient):
-    """Redis SentinelClient that refuses to cache large objects
-
-    Size applies to the literal cached value, which is _after_ serialization and
-    compression.
-
-    Set size limit with MAX_ENCODED_VALUE_LEN in the OPTIONS dict. Defaults to
-    1 MB.
-    """
-    pass
+            super().set(key, value, timeout, version)
+        except MemcacheServerError as err:
+            if "object too large for cache" in str(err):
+                log(f"Memcache failed to cache large object for {key}")
+            else:
+                raise