ci: nearly deployable helm chart (wip)

.editorconfig

@@ -50,3 +50,9 @@ indent_size = 2
 
 [ietf/**.html]
 insert_final_newline = false
+
+# Settings for Helm chart yaml
+# ---------------------------------------------------------
+# Use 2-space indents
+[helm/**.yaml]
+indent_size = 2

helm/settings_local.py

@@ -0,0 +1,121 @@
+# Copyright The IETF Trust 2007-2024, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+from ietf.settings import *                                          # pyflakes:ignore
+
+ALLOWED_HOSTS = ['*']
+
+DATABASES = {
+    "default": {
+        "HOST": os.environ.get("DBHOST", "db"),
+        "PORT": os.environ.get("DBPORT", "5432"),
+        "NAME": os.environ.get("DBNAME", "datatracker"),
+        "ENGINE": "django.db.backends.postgresql",
+        "USER": os.environ.get("DBUSER", "django"),
+        "PASSWORD": os.environ.get("DBPASS", ""),
+    },
+}
+
+SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY")
+
+CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL")
+
+IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
+IDSUBMIT_REPOSITORY_PATH = "/test/id/"
+IDSUBMIT_STAGING_PATH = "/test/staging/"
+
+AGENDA_PATH = "/assets/www6s/proceedings/"
+MEETINGHOST_LOGO_PATH = AGENDA_PATH
+
+USING_DEBUG_EMAIL_SERVER=True
+EMAIL_HOST= "localhost"
+EMAIL_PORT=2025
+
+MEDIA_BASE_DIR = "/assets"
+MEDIA_ROOT = MEDIA_BASE_DIR + "/media/"
+MEDIA_URL = "/media/"
+
+PHOTOS_DIRNAME = "photo"
+PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME
+
+SUBMIT_YANG_CATALOG_MODEL_DIR = "/assets/ietf-ftp/yang/catalogmod/"
+SUBMIT_YANG_DRAFT_MODEL_DIR = "/assets/ietf-ftp/yang/draftmod/"
+SUBMIT_YANG_INVAL_MODEL_DIR = "/assets/ietf-ftp/yang/invalmod/"
+SUBMIT_YANG_IANA_MODEL_DIR = "/assets/ietf-ftp/yang/ianamod/"
+SUBMIT_YANG_RFC_MODEL_DIR = "/assets/ietf-ftp/yang/rfcmod/"
+
+# Set INTERNAL_IPS for use within Docker. See https://knasmueller.net/fix-djangos-debug-toolbar-not-showing-inside-docker
+import socket
+hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
+INTERNAL_IPS = [".".join(ip.split(".")[:-1] + ["1"]) for ip in ips]
+
+# DEV_TEMPLATE_CONTEXT_PROCESSORS = [
+#    'ietf.context_processors.sql_debug',
+# ]
+
+DOCUMENT_PATH_PATTERN = "/assets/ietfdata/doc/{doc.type_id}/"
+INTERNET_DRAFT_PATH = "/assets/ietf-ftp/internet-drafts/"
+RFC_PATH = "/assets/ietf-ftp/rfc/"
+CHARTER_PATH = "/assets/ietf-ftp/charter/"
+BOFREQ_PATH = "/assets/ietf-ftp/bofreq/"
+CONFLICT_REVIEW_PATH = "/assets/ietf-ftp/conflict-reviews/"
+STATUS_CHANGE_PATH = "/assets/ietf-ftp/status-changes/"
+INTERNET_DRAFT_ARCHIVE_DIR = "/assets/archive/id"
+INTERNET_ALL_DRAFTS_ARCHIVE_DIR = "/assets/archive/id"
+BIBXML_BASE_PATH = "/assets/ietfdata/derived/bibxml"
+IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
+
+NOMCOM_PUBLIC_KEYS_DIR = "data/nomcom_keys/public_keys/"
+SLIDE_STAGING_PATH = "/test/staging/"
+
+# todo check that de-gfm is in place
+DE_GFM_BINARY = "/usr/local/bin/de-gfm"
+
+# OIDC configuration
+SITE_URL = os.environ.get("OIDC_SITE_URL")
+
+# todo: parameterize memcached url in settings.py
+MEMCACHED_HOST = os.environ.get("DTH_DATATRACKER_MEMCACHED_SERVICE_HOST", "127.0.0.1")
+MEMCACHED_PORT = os.environ.get("DTH_DATATRACKER_MEMCACHED_SERVICE_PORT", "11211")
+from ietf import __version__
+CACHES = {
+    "default": {
+        "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+        "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+        "VERSION": __version__,
+        "KEY_PREFIX": "ietf:dt",
+        "KEY_FUNCTION": lambda key, key_prefix, version: (
+            f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+        ),
+    },
+    "sessions": {
+        "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+        "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+        # No release-specific VERSION setting.
+        "KEY_PREFIX": "ietf:dt",
+    },
+    "htmlized": {
+        "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
+        "LOCATION": "/a/cache/datatracker/htmlized",
+        "OPTIONS": {
+            "MAX_ENTRIES": 100000,  # 100,000
+        },
+    },
+    "pdfized": {
+        "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
+        "LOCATION": "/a/cache/datatracker/pdfized",
+        "OPTIONS": {
+            "MAX_ENTRIES": 100000,  # 100,000
+        },
+    },
+    "slowpages": {
+        "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
+        "LOCATION": "/a/cache/datatracker/slowpages",
+        "OPTIONS": {
+            "MAX_ENTRIES": 5000,
+        },
+    },
+}
+
+# Normally only set for debug, but needed until we have a real FS
+DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json')

helm/templates/_helpers.tpl

@@ -176,4 +176,4 @@ Create the name of the service account to use
 {{- else -}}
   {{ default "default" .Values.serviceAccounts.memcached.name }}
 {{- end -}}
-{{- end }}
+{{- end }}

helm/templates/configmap.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: django-configmap
+data:
+  settings_local.py: |-
+    {{- .Files.Get "settings_local.py" | indent 4 }}

helm/templates/deployments/beat.yaml

@@ -1,67 +1,72 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "datatracker.fullname" . }}
-  labels:
-    {{- include "datatracker.labels" . | nindent 4 }}
-spec:
-  replicas: {{ .Values.replicaCount }}
-  revisionHistoryLimit: {{ .Values.revisionHistoryLimit }}
-  selector:
-    matchLabels:
-      {{- include "datatracker.selectorLabels" . | nindent 6 }}
-  template:
-    metadata:
-      labels:
-        {{- include "datatracker.selectorLabels" . | nindent 8 }}
-    spec:
-    {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-    {{- end }}
-      serviceAccountName: {{ include "datatracker.serviceAccountName" . }}
-      securityContext:
-        {{- toYaml .Values.podSecurityContext | nindent 8 }}
-      containers:
-        - name: {{ .Chart.Name }}
-          securityContext:
-            {{- toYaml .Values.securityContext | nindent 12 }}
-          image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}"
-          imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }}
-          env:
-          CONTAINER_ROLE: beat
-            {{- if .Values.env }}
-            {{- toYaml .Values.env | nindent 12 }}
-            {{- end }}
-    {{- with .Values.volumeMounts }}
-          volumeMounts:
-            {{- toYaml . | nindent 12 }}
-    {{- end }}
-          ports:
-            - name: http
-              containerPort: 8000
-              protocol: TCP
-          livenessProbe:
-            {{- toYaml .Values.livenessProbe | nindent 12 }}
-          readinessProbe:
-            {{- toYaml .Values.readinessProbe | nindent 12 }}
-          startupProbe:
-            {{- toYaml .Values.startupProbe | nindent 12 }}
-          resources:
-            {{- toYaml .Values.resources | nindent 12 }}
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-    {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-    {{- end }}
-    {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-    {{- end }}
-    {{- with .Values.volumes }}
-      volumes:
-        {{- toYaml . | nindent 8 }}
-    {{- end }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "datatracker.beat.fullname" . }}
+  labels:
+    {{- include "datatracker.labels" . | nindent 4 }}
+spec:
+  {{- $podValues := .Values.beat }}
+  replicas: {{ $podValues.replicaCount }}
+  revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }}
+  selector:
+    matchLabels:
+      {{- include "datatracker.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "datatracker.selectorLabels" . | nindent 8 }}
+    spec:
+    {{- with $podValues.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      serviceAccountName: {{ include "datatracker.serviceAccountName.beat" . }}
+      securityContext:
+        {{- toYaml $podValues.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml $podValues.securityContext | nindent 12 }}
+          image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}"
+          imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }}
+          env:
+            - name: "CONTAINER_ROLE"
+              value: "beat"
+            {{- if .Values.env }}
+            {{- range $key, $val := .Values.env }}
+            - name: {{ $key | quote }}
+              value: {{ $val | quote }}
+            {{- end }}
+            {{- end }}
+    {{- with $podValues.volumeMounts }}
+          volumeMounts:
+            {{- toYaml . | nindent 12 }}
+    {{- end }}
+          ports:
+            - name: http
+              containerPort: 8000
+              protocol: TCP
+          livenessProbe:
+            {{- toYaml $podValues.livenessProbe | nindent 12 }}
+          readinessProbe:
+            {{- toYaml $podValues.readinessProbe | nindent 12 }}
+          startupProbe:
+            {{- toYaml $podValues.startupProbe | nindent 12 }}
+          resources:
+            {{- toYaml $podValues.resources | nindent 12 }}
+      {{- with $podValues.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    {{- with $podValues.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with $podValues.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with $podValues.volumes }}
+      volumes:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}

helm/templates/deployments/celery.yaml

@@ -1,12 +1,13 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ include "datatracker.fullname" . }}
+  name: {{ include "datatracker.celery.fullname" . }}
   labels:
     {{- include "datatracker.labels" . | nindent 4 }}
 spec:
-  replicas: {{ .Values.replicaCount }}
-  revisionHistoryLimit: {{ .Values.revisionHistoryLimit }}
+  {{- $podValues := .Values.celery }}
+  replicas: {{ $podValues.replicaCount }}
+  revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }}
   selector:
     matchLabels:
       {{- include "datatracker.selectorLabels" . | nindent 6 }}
@@ -15,25 +16,29 @@ spec:
       labels:
         {{- include "datatracker.selectorLabels" . | nindent 8 }}
     spec:
-    {{- with .Values.imagePullSecrets }}
+    {{- with $podValues.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
     {{- end }}
-      serviceAccountName: {{ include "datatracker.serviceAccountName" . }}
+      serviceAccountName: {{ include "datatracker.serviceAccountName.celery" . }}
       securityContext:
-        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+        {{- toYaml $podValues.podSecurityContext | nindent 8 }}
       containers:
         - name: {{ .Chart.Name }}
           securityContext:
-            {{- toYaml .Values.securityContext | nindent 12 }}
-          image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}"
-          imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }}
+            {{- toYaml $podValues.securityContext | nindent 12 }}
+          image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}"
+          imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }}
           env:
-          CONTAINER_ROLE: celery
+            - name: "CONTAINER_ROLE"
+              value: "celery"
             {{- if .Values.env }}
-            {{- toYaml .Values.env | nindent 12 }}
+            {{- range $key, $val := .Values.env }}
+            - name: {{ $key | quote }}
+              value: {{ $val | quote }}
             {{- end }}
-    {{- with .Values.volumeMounts }}
+            {{- end }}
+    {{- with $podValues.volumeMounts }}
           volumeMounts:
             {{- toYaml . | nindent 12 }}
     {{- end }}
@@ -42,26 +47,26 @@ spec:
               containerPort: 8000
               protocol: TCP
           livenessProbe:
-            {{- toYaml .Values.livenessProbe | nindent 12 }}
+            {{- toYaml $podValues.livenessProbe | nindent 12 }}
           readinessProbe:
-            {{- toYaml .Values.readinessProbe | nindent 12 }}
+            {{- toYaml $podValues.readinessProbe | nindent 12 }}
           startupProbe:
-            {{- toYaml .Values.startupProbe | nindent 12 }}
+            {{- toYaml $podValues.startupProbe | nindent 12 }}
           resources:
-            {{- toYaml .Values.resources | nindent 12 }}
-      {{- with .Values.nodeSelector }}
+            {{- toYaml $podValues.resources | nindent 12 }}
+      {{- with $podValues.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-    {{- with .Values.affinity }}
+    {{- with $podValues.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
     {{- end }}
-    {{- with .Values.tolerations }}
+    {{- with $podValues.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
     {{- end }}
-    {{- with .Values.volumes }}
+    {{- with $podValues.volumes }}
       volumes:
         {{- toYaml . | nindent 8 }}
     {{- end }}