From ac1567b564b0868fabeb0bceafd24d9b997d9749 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 14 Jul 2025 13:38:59 -0300 Subject: [PATCH 1/5] chore(dev): use redis for dev sessions Adds a redis container with a persistent volume and uses this for sessions instead of the database. Results in the same general behavior as before - logins should be persistent until the docker compose stack is torn down. --- docker-compose.yml | 7 +++++++ ietf/settings.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 073d04b896a..f171fb261bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -90,6 +90,12 @@ services: - .:/workspace - app-assets:/assets + redis: + image: redis:8 + command: ['redis-server', '--save', '10', '1', '--loglevel', 'warning'] + volumes: + - redis:/data + replicator: build: context: . @@ -169,3 +175,4 @@ volumes: app-assets: minio-data: blobdb-data: + redis: diff --git a/ietf/settings.py b/ietf/settings.py index 95f2ffefd76..bbdb77fb2a1 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1463,7 +1463,8 @@ def skip_unreadable_post(record): ), }, "sessions": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": "redis://redis:6379", }, "htmlized": { "BACKEND": "django.core.cache.backends.dummy.DummyCache", @@ -1509,7 +1510,6 @@ def skip_unreadable_post(record): loaders = TEMPLATES[0]['OPTIONS']['loaders'] loaders = tuple(l for e in loaders for l in (e[1] if isinstance(e, tuple) and "cached.Loader" in e[0] else (e,))) TEMPLATES[0]['OPTIONS']['loaders'] = loaders - SESSION_ENGINE = "django.contrib.sessions.backends.db" if 'SECRET_KEY' not in locals(): SECRET_KEY = 'PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHka' From 65ea1ec4375c3730d685086754156ef018d13b26 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 27 May 2026 13:19:02 -0300 Subject: [PATCH 2/5] chore(deps): install redis/hiredis python pkgs --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 31e8ea69d1e..5f237812417 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,6 +39,7 @@ drf-standardized-errors[openapi] >= 0.15.0 factory-boy>=3.3.3 gunicorn>=23.0.0 hashids>=1.3.1 +hiredis>=3.3.1 html2text>=2025.4.15 # Used only to clean comment field of secr/sreq html5lib>=1.1 # Only used in tests httpx>=0.28.1 # Indirect req of typesense, but we import and refer to exceptions @@ -75,6 +76,7 @@ python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=2.0.0 # from TastyPie pytz==2025.2 # Pinned as changes need to be vetted for their effect on Meeting fields +redis>=7.4.0 types-pytz==2025.2.0.20251108 # match pytz version typesense>=2.0.0 requests>=2.32.4 From 76108d744272265755bce2bcbce2fc0fe12ac88e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 27 May 2026 16:44:42 -0300 Subject: [PATCH 3/5] chore: remove memcached (mostly) Still keeps some config references and a custom cache backend --- docker/base.Dockerfile | 2 - ietf/settings.py | 16 ++++---- k8s/kustomization.yaml | 1 - k8s/memcached.yaml | 88 ------------------------------------------ k8s/settings_local.py | 4 +- requirements.txt | 1 - 6 files changed, 10 insertions(+), 102 deletions(-) delete mode 100644 k8s/memcached.yaml diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 25016360494..7bf4263b382 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -59,12 +59,10 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends libxtst6 \ libmagic-dev \ libmariadb-dev \ - libmemcached-tools \ libyang2-tools \ locales \ make \ mariadb-client \ - memcached \ nano \ netcat-traditional \ nodejs \ diff --git a/ietf/settings.py b/ietf/settings.py index bbdb77fb2a1..13dbfb5c152 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1434,16 +1434,16 @@ def skip_unreadable_post(record): CACHES = { "default": { "BACKEND": "django.core.cache.backends.dummy.DummyCache", - #'BACKEND': 'ietf.utils.cache.LenientMemcacheCache', - #'LOCATION': '127.0.0.1:11211', + # "BACKEND": "django.core.cache.backends.redis.RedisCache", + # "LOCATION": "redis://redis:6379", #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', "VERSION": __version__, "KEY_PREFIX": "ietf:dt", }, "agenda": { "BACKEND": "django.core.cache.backends.dummy.DummyCache", - # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - # "LOCATION": "127.0.0.1:11211", + # "BACKEND": "django.core.cache.backends.redis.RedisCache", + # "LOCATION": "redis://redis:6379", # No release-specific VERSION setting. "KEY_PREFIX": "ietf:dt:agenda", # Key function is default except with sha384-encoded key @@ -1453,8 +1453,8 @@ def skip_unreadable_post(record): }, "proceedings": { "BACKEND": "django.core.cache.backends.dummy.DummyCache", - # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - # "LOCATION": "127.0.0.1:11211", + #"BACKEND": "django.core.cache.backends.redis.RedisCache", + #"LOCATION": "redis://redis:6379", # No release-specific VERSION setting. "KEY_PREFIX": "ietf:dt:proceedings", # Key function is default except with sha384-encoded key @@ -1491,8 +1491,8 @@ def skip_unreadable_post(record): }, }, "celery-results": { - "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", - "LOCATION": "app:11211", + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": "redis://redis:6379", "KEY_PREFIX": "ietf:celery", }, } diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 769cb03517e..b1e278a9149 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -12,6 +12,5 @@ resources: - beat.yaml - celery.yaml - datatracker.yaml - - memcached.yaml - rabbitmq.yaml - replicator.yaml diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml deleted file mode 100644 index 68b732d7459..00000000000 --- a/k8s/memcached.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: memcached -spec: - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - app: memcached - template: - metadata: - labels: - app: memcached - spec: - securityContext: - runAsNonRoot: true - containers: - # ----------------------------------------------------- - # Memcached - # ----------------------------------------------------- - - image: "memcached:1.6-alpine" - imagePullPolicy: IfNotPresent - args: ["-m", "1024"] - name: memcached - ports: - - name: memcached - containerPort: 11211 - protocol: TCP - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - # memcached image sets up uid/gid 11211 - runAsUser: 11211 - runAsGroup: 11211 - resources: - requests: - cpu: 100m - memory: 100Mi - # ----------------------------------------------------- - # Memcached Exporter for Prometheus - # ----------------------------------------------------- - - image: "quay.io/prometheus/memcached-exporter:v0.14.3" - imagePullPolicy: IfNotPresent - name: memcached-exporter - ports: - - name: metrics - containerPort: 9150 - protocol: TCP - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 65534 # nobody - runAsGroup: 65534 # nobody - resources: - requests: - cpu: 10m - memory: 20Mi - dnsPolicy: ClusterFirst - restartPolicy: Always - terminationGracePeriodSeconds: 30 ---- -apiVersion: v1 -kind: Service -metadata: - name: memcached - annotations: - k8s.grafana.com/scrape: "true" # this is not a bool - k8s.grafana.com/metrics.portName: "metrics" -spec: - type: ClusterIP - ports: - - port: 11211 - targetPort: memcached - protocol: TCP - name: memcached - - port: 9150 - targetPort: metrics - protocol: TCP - name: metrics - selector: - app: memcached diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 402f89787b1..2d64e5f1230 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -368,8 +368,8 @@ def _multiline_to_list(s): }, }, "celery-results": { - "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": f"{REDIS_HOST}:{REDIS_PORT}", "KEY_PREFIX": "ietf:celery", }, } diff --git a/requirements.txt b/requirements.txt index 5f237812417..bc59e8e17c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -73,7 +73,6 @@ python-dateutil>=2.9.0 types-python-dateutil>=2.9.0 python-json-logger>=3.3.0 python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form failures -pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=2.0.0 # from TastyPie pytz==2025.2 # Pinned as changes need to be vetted for their effect on Meeting fields redis>=7.4.0 From 00f1fc39c10fbb097f70fa76eb9ebc7feaa2e3e1 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 27 May 2026 17:10:04 -0300 Subject: [PATCH 4/5] chore: switch to django-redis for sentinel support --- ietf/settings.py | 37 ++++++++++++++++++++++++++----------- requirements.txt | 7 +++---- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index 13dbfb5c152..dc639529cee 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1434,16 +1434,22 @@ def skip_unreadable_post(record): CACHES = { "default": { "BACKEND": "django.core.cache.backends.dummy.DummyCache", - # "BACKEND": "django.core.cache.backends.redis.RedisCache", - # "LOCATION": "redis://redis:6379", - #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + # "BACKEND": "django_redis.cache.RedisCache", + # "LOCATION": "redis://redis:6379/0", + # "OPTIONS": { + # "CLIENT_CLASS": "django_redis.client.DefaultClient", + # }, + # "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", "VERSION": __version__, "KEY_PREFIX": "ietf:dt", }, "agenda": { "BACKEND": "django.core.cache.backends.dummy.DummyCache", - # "BACKEND": "django.core.cache.backends.redis.RedisCache", - # "LOCATION": "redis://redis:6379", + # "BACKEND": "django_redis.cache.RedisCache", + # "LOCATION": "redis://redis:6379/0", + # "OPTIONS": { + # "CLIENT_CLASS": "django_redis.client.DefaultClient", + # }, # No release-specific VERSION setting. "KEY_PREFIX": "ietf:dt:agenda", # Key function is default except with sha384-encoded key @@ -1453,8 +1459,11 @@ def skip_unreadable_post(record): }, "proceedings": { "BACKEND": "django.core.cache.backends.dummy.DummyCache", - #"BACKEND": "django.core.cache.backends.redis.RedisCache", - #"LOCATION": "redis://redis:6379", + # "BACKEND": "django_redis.cache.RedisCache", + # "LOCATION": "redis://redis:6379/0", + # "OPTIONS": { + # "CLIENT_CLASS": "django_redis.client.DefaultClient", + # }, # No release-specific VERSION setting. "KEY_PREFIX": "ietf:dt:proceedings", # Key function is default except with sha384-encoded key @@ -1463,8 +1472,11 @@ def skip_unreadable_post(record): ), }, "sessions": { - "BACKEND": "django.core.cache.backends.redis.RedisCache", - "LOCATION": "redis://redis:6379", + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": "redis://redis:6379/0", + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + }, }, "htmlized": { "BACKEND": "django.core.cache.backends.dummy.DummyCache", @@ -1491,8 +1503,11 @@ def skip_unreadable_post(record): }, }, "celery-results": { - "BACKEND": "django.core.cache.backends.redis.RedisCache", - "LOCATION": "redis://redis:6379", + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": "redis://redis:6379/0", + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + }, "KEY_PREFIX": "ietf:celery", }, } diff --git a/requirements.txt b/requirements.txt index bc59e8e17c3..bf7d1d26e0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,6 +25,7 @@ django-debug-toolbar>=6.0.0 django-filter>=24.3 django-markup>=1.10 # Limited use - need to reconcile against direct use of markdown django-oidc-provider==0.8.2 # 0.8.3 changes logout flow and claim return +django-redis>=6.0.0 django-simple-history>=3.10.1 django-storages>=1.14.6 django-stubs>=4.2.7,<5 # The django-stubs version used determines the the mypy version indicated below @@ -39,7 +40,6 @@ drf-standardized-errors[openapi] >= 0.15.0 factory-boy>=3.3.3 gunicorn>=23.0.0 hashids>=1.3.1 -hiredis>=3.3.1 html2text>=2025.4.15 # Used only to clean comment field of secr/sreq html5lib>=1.1 # Only used in tests httpx>=0.28.1 # Indirect req of typesense, but we import and refer to exceptions @@ -59,8 +59,8 @@ oic>=1.7.0 # Used only by tests opentelemetry-sdk>=1.38.0 opentelemetry-instrumentation-django>=0.59b0 opentelemetry-instrumentation-psycopg2>=0.59b0 -opentelemetry-instrumentation-pymemcache>=0.59b0 opentelemetry-instrumentation-requests>=0.59b0 +opentelemetry-instrumentation-redis>=0.63b1 opentelemetry-exporter-otlp-proto-http>=1.38.0 pillow>=11.3.0 psycopg2>=2.9.10 @@ -75,9 +75,7 @@ python-json-logger>=3.3.0 python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form failures python-mimeparse>=2.0.0 # from TastyPie pytz==2025.2 # Pinned as changes need to be vetted for their effect on Meeting fields -redis>=7.4.0 types-pytz==2025.2.0.20251108 # match pytz version -typesense>=2.0.0 requests>=2.32.4 types-requests>=2.32.4 requests-mock>=1.12.1 @@ -87,6 +85,7 @@ selenium>=4.34.2 tblib>=3.1.0 # So that the django test runner provides tracebacks tlds>=2022042700 # Used to teach bleach about which TLDs currently exist tqdm>=4.67.1 +typesense>=2.0.0 unidecode>=1.4.0 urllib3>=2.5.0 weasyprint>=66.0 From 6992cfbcddd8d7724cd33b6b1ce4eb6fc903fccf Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 27 May 2026 17:57:16 -0300 Subject: [PATCH 5/5] chore: production redis cache config Uses sentinel. Untested. --- ietf/settings.py | 218 +++++++++++++++--------------------------- k8s/settings_local.py | 38 ++++---- 2 files changed, 96 insertions(+), 160 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index dc639529cee..dc68fdcaf06 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1364,153 +1364,87 @@ def skip_unreadable_post(record): if "CACHES" not in locals(): if SERVER_MODE == "production": - MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") - MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") - CACHES = { - "default": { - "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", - "VERSION": __version__, - "KEY_PREFIX": "ietf:dt", - # Key function is default except with sha384-encoded key - "KEY_FUNCTION": lambda key, key_prefix, version: ( - f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" - ), - }, - "agenda": { - "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", - # No release-specific VERSION setting. - "KEY_PREFIX": "ietf:dt:agenda", - # Key function is default except with sha384-encoded key - "KEY_FUNCTION": lambda key, key_prefix, version: ( - f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" - ), - }, - "proceedings": { - "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", - # No release-specific VERSION setting. - "KEY_PREFIX": "ietf:dt:proceedings", - # Key function is default except with sha384-encoded key - "KEY_FUNCTION": lambda key, key_prefix, version: ( - f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" - ), - }, - "sessions": { - "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", - # No release-specific VERSION setting. - "KEY_PREFIX": "ietf:dt", - }, - "htmlized": { - "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/a/cache/datatracker/htmlized", - "OPTIONS": { - "MAX_ENTRIES": 100000, # 100,000 - }, - }, - "pdfized": { - "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/a/cache/datatracker/pdfized", - "OPTIONS": { - "MAX_ENTRIES": 100000, # 100,000 - }, - }, - "slowpages": { - "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/a/cache/datatracker/slowpages", - "OPTIONS": { - "MAX_ENTRIES": 5000, - }, - }, - "celery-results": { - "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", - "KEY_PREFIX": "ietf:celery", - }, - } - else: - CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", - # "BACKEND": "django_redis.cache.RedisCache", - # "LOCATION": "redis://redis:6379/0", - # "OPTIONS": { - # "CLIENT_CLASS": "django_redis.client.DefaultClient", - # }, - # "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "VERSION": __version__, - "KEY_PREFIX": "ietf:dt", - }, - "agenda": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", - # "BACKEND": "django_redis.cache.RedisCache", - # "LOCATION": "redis://redis:6379/0", - # "OPTIONS": { - # "CLIENT_CLASS": "django_redis.client.DefaultClient", - # }, - # No release-specific VERSION setting. - "KEY_PREFIX": "ietf:dt:agenda", - # Key function is default except with sha384-encoded key - "KEY_FUNCTION": lambda key, key_prefix, version: ( - f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" - ), - }, - "proceedings": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", - # "BACKEND": "django_redis.cache.RedisCache", - # "LOCATION": "redis://redis:6379/0", - # "OPTIONS": { - # "CLIENT_CLASS": "django_redis.client.DefaultClient", - # }, - # No release-specific VERSION setting. - "KEY_PREFIX": "ietf:dt:proceedings", - # Key function is default except with sha384-encoded key - "KEY_FUNCTION": lambda key, key_prefix, version: ( - f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" - ), - }, - "sessions": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://redis:6379/0", - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - }, + raise RuntimeError("Must set CACHES in settings_local for production mode") + CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + # "BACKEND": "django_redis.cache.RedisCache", + # "LOCATION": "redis://redis:6379/0", + # "OPTIONS": { + # "CLIENT_CLASS": "django_redis.client.DefaultClient", + # }, + # "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "VERSION": __version__, + "KEY_PREFIX": "ietf:dt", + }, + "agenda": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + # "BACKEND": "django_redis.cache.RedisCache", + # "LOCATION": "redis://redis:6379/0", + # "OPTIONS": { + # "CLIENT_CLASS": "django_redis.client.DefaultClient", + # }, + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "proceedings": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + # "BACKEND": "django_redis.cache.RedisCache", + # "LOCATION": "redis://redis:6379/0", + # "OPTIONS": { + # "CLIENT_CLASS": "django_redis.client.DefaultClient", + # }, + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:proceedings", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "sessions": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": "redis://redis:6379/0", + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", }, - "htmlized": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", - #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - "LOCATION": "/var/cache/datatracker/htmlized", - "OPTIONS": { - "MAX_ENTRIES": 1000, - }, + }, + "htmlized": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + "LOCATION": "/var/cache/datatracker/htmlized", + "OPTIONS": { + "MAX_ENTRIES": 1000, }, - "pdfized": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", - #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - "LOCATION": "/var/cache/datatracker/pdfized", - "OPTIONS": { - "MAX_ENTRIES": 1000, - }, + }, + "pdfized": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + "LOCATION": "/var/cache/datatracker/pdfized", + "OPTIONS": { + "MAX_ENTRIES": 1000, }, - "slowpages": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", - #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - "LOCATION": "/var/cache/datatracker/", - "OPTIONS": { - "MAX_ENTRIES": 5000, - }, + }, + "slowpages": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + "LOCATION": "/var/cache/datatracker/", + "OPTIONS": { + "MAX_ENTRIES": 5000, }, - "celery-results": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://redis:6379/0", - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - }, - "KEY_PREFIX": "ietf:celery", + }, + "celery-results": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": "redis://redis:6379/0", + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", }, - } + "KEY_PREFIX": "ietf:celery", + }, + } PUBLISH_IPR_STATES = ['posted', 'removed', 'removed_objfalse'] diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 2d64e5f1230..942d3f31345 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -304,15 +304,25 @@ def _multiline_to_list(s): DE_GFM_BINARY = "/usr/local/bin/de-gfm" IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" -# Duplicating production cache from settings.py and using it whether we're in production mode or not -MEMCACHED_HOST = os.environ.get("DT_MEMCACHED_SERVICE_HOST", "127.0.0.1") -MEMCACHED_PORT = os.environ.get("DT_MEMCACHED_SERVICE_PORT", "11211") from ietf import __version__ +# Common config for redis caches +REDIS_SENTINEL_SERVICE = os.environ.get("DATATRACKER_REDIS_SENTINEL_HOST") +REDIS_SENTINEL_PORT = os.environ.get("DATATRACKER_REDIS_SENTINEL_PORT", "26379") +DJANGO_REDIS_CONNECTION_FACTORY = "django_redis.pool.SentinelConnectionFactory" +REDIS_CACHE_CONFIG_COMMON = { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": "redis://myMaster/0", + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.SentinelClient", + "SENTINELS": [(REDIS_SENTINEL_SERVICE, REDIS_SENTINEL_PORT)], + "SENTINEL_KWARGS": {}, + "CONNECTION_POOL_CLASS": "redis.sentinel.SentinelConnectionPool", + }, +} + CACHES = { - "default": { - "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + "default": REDIS_CACHE_CONFIG_COMMON | { "VERSION": __version__, "KEY_PREFIX": "ietf:dt", # Key function is default except with sha384-encoded key @@ -320,9 +330,7 @@ def _multiline_to_list(s): f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), }, - "agenda": { - "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + "agenda": REDIS_CACHE_CONFIG_COMMON | { # No release-specific VERSION setting. "KEY_PREFIX": "ietf:dt:agenda", # Key function is default except with sha384-encoded key @@ -330,9 +338,7 @@ def _multiline_to_list(s): f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), }, - "proceedings": { - "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + "proceedings": REDIS_CACHE_CONFIG_COMMON | { # No release-specific VERSION setting. "KEY_PREFIX": "ietf:dt:proceedings", # Key function is default except with sha384-encoded key @@ -340,9 +346,7 @@ def _multiline_to_list(s): f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), }, - "sessions": { - "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + "sessions": REDIS_CACHE_CONFIG_COMMON | { # No release-specific VERSION setting. "KEY_PREFIX": "ietf:dt", }, @@ -367,9 +371,7 @@ def _multiline_to_list(s): "MAX_ENTRIES": 5000, }, }, - "celery-results": { - "BACKEND": "django.core.cache.backends.redis.RedisCache", - "LOCATION": f"{REDIS_HOST}:{REDIS_PORT}", + "celery-results": REDIS_CACHE_CONFIG_COMMON | { "KEY_PREFIX": "ietf:celery", }, }