From 7cfe5afb3e0f29324c3129c0c2545ba6264ba1b1 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 16 Jul 2024 16:02:03 -0300 Subject: [PATCH 1/7] fix: silence nginx healthcheck logs --- k8s/nginx-auth.conf | 1 + k8s/nginx-datatracker.conf | 1 + 2 files changed, 2 insertions(+) diff --git a/k8s/nginx-auth.conf b/k8s/nginx-auth.conf index 4cbc8a0a51..f3d5770def 100644 --- a/k8s/nginx-auth.conf +++ b/k8s/nginx-auth.conf @@ -5,6 +5,7 @@ server { # Note that regex location matches take priority over non-regex "prefix" matches. Use regexes so that # our deny all rule does not squelch the other locations. location ~ ^/health/nginx$ { + access_log off; return 200; } diff --git a/k8s/nginx-datatracker.conf b/k8s/nginx-datatracker.conf index 63c985463c..bf753968f7 100644 --- a/k8s/nginx-datatracker.conf +++ b/k8s/nginx-datatracker.conf @@ -3,6 +3,7 @@ server { server_name _; location /health/nginx { + access_log off; return 200; } From 9533c390e049c51e108200ba305576331a2a6cd8 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 16 Jul 2024 16:10:22 -0300 Subject: [PATCH 2/7] fix: nginx logs in JSON --- k8s/auth.yaml | 3 +++ k8s/datatracker.yaml | 3 +++ k8s/kustomization.yaml | 1 + k8s/nginx.conf | 52 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+) create mode 100644 k8s/nginx.conf diff --git a/k8s/auth.yaml b/k8s/auth.yaml index 66627ed450..ae5509d806 100644 --- a/k8s/auth.yaml +++ b/k8s/auth.yaml @@ -80,6 +80,9 @@ spec: volumeMounts: - name: nginx-tmp mountPath: /tmp + - name: dt-cfg + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf - name: dt-cfg mountPath: /etc/nginx/conf.d/auth.conf subPath: nginx-auth.conf diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 9e1ead1a90..8a9fbb1112 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -80,6 +80,9 @@ spec: volumeMounts: - name: nginx-tmp mountPath: /tmp + - name: dt-cfg + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf - name: dt-cfg mountPath: /etc/nginx/conf.d/datatracker.conf subPath: nginx-datatracker.conf diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index ba8b8a5826..df24143aef 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -3,6 +3,7 @@ namePrefix: dt- configMapGenerator: - name: files-cfgmap files: + - nginx.conf - nginx-auth.conf - nginx-datatracker.conf - settings_local.py diff --git a/k8s/nginx.conf b/k8s/nginx.conf new file mode 100644 index 0000000000..403c515c7e --- /dev/null +++ b/k8s/nginx.conf @@ -0,0 +1,52 @@ +# nginx configuration +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /tmp/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + proxy_temp_path /tmp/proxy_temp; + client_body_temp_path /tmp/client_temp; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main escape=json + '{' + '"time":"$${keepempty}time_iso8601",' + '"remote_ip":"$${keepempty}remote_addr",' + '"request":"$${keepempty}request",' + '"host":"$${keepempty}host",' + '"path":"$${keepempty}request_uri",' + '"method":"$${keepempty}request_method",' + '"status":"$${keepempty}status",' + '"len_bytes":"$${keepempty}body_bytes_sent",' + '"duration_ms":"$${keepempty}request_time",' + '"referer":"$${keepempty}http_referer",' + '"user_agent":"$${keepempty}http_user_agent",' + '"x_forwarded_for":"$${keepempty}http_x_forwarded_for",' + '"cf_connecting_ip":"$${keepempty}http_cf_connecting_ip",' + '"cf_connecting_ipv6":"$${keepempty}http_cf_connecting_ipv6",' + '"cf_ray":"$${keepempty}cf_ray"' + '}'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; +} From ef037e556275573f4c3bf07cf82876bc1595245c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 16 Jul 2024 16:57:52 -0300 Subject: [PATCH 3/7] fix: typos in nginx conf --- k8s/nginx-auth.conf | 2 +- k8s/nginx-datatracker.conf | 2 +- k8s/nginx.conf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/nginx-auth.conf b/k8s/nginx-auth.conf index f3d5770def..f3f823dd15 100644 --- a/k8s/nginx-auth.conf +++ b/k8s/nginx-auth.conf @@ -27,7 +27,7 @@ server { add_header Content-Security-Policy "default-src 'self' 'unsafe-inline' data: https://datatracker.ietf.org/ https://www.ietf.org/ http://ietf.org/ https://analytics.ietf.org https://static.ietf.org; frame-ancestors 'self' ietf.org *.ietf.org meetecho.com *.meetecho.com gather.town *.gather.town"; proxy_set_header Host $${keepempty}host; proxy_set_header Connection close; - proxy_set_header X-Request-Start "t=${msec}"; + proxy_set_header X-Request-Start "t=$${keepempty}msec"; proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for; proxy_set_header X-Real-IP $${keepempty}remote_addr; proxy_pass http://localhost:8000; diff --git a/k8s/nginx-datatracker.conf b/k8s/nginx-datatracker.conf index bf753968f7..d3a73bac9b 100644 --- a/k8s/nginx-datatracker.conf +++ b/k8s/nginx-datatracker.conf @@ -16,7 +16,7 @@ server { add_header Content-Security-Policy "default-src 'self' 'unsafe-inline' data: https://datatracker.ietf.org/ https://www.ietf.org/ http://ietf.org/ https://analytics.ietf.org https://static.ietf.org; frame-ancestors 'self' ietf.org *.ietf.org meetecho.com *.meetecho.com"; proxy_set_header Host $${keepempty}host; proxy_set_header Connection close; - proxy_set_header X-Request-Start "t=${msec}"; + proxy_set_header X-Request-Start "t=$${keepempty}msec"; proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for; proxy_set_header X-Real-IP $${keepempty}remote_addr; proxy_pass http://localhost:8000; diff --git a/k8s/nginx.conf b/k8s/nginx.conf index 403c515c7e..00cdbbdf4e 100644 --- a/k8s/nginx.conf +++ b/k8s/nginx.conf @@ -36,7 +36,7 @@ http { '"x_forwarded_for":"$${keepempty}http_x_forwarded_for",' '"cf_connecting_ip":"$${keepempty}http_cf_connecting_ip",' '"cf_connecting_ipv6":"$${keepempty}http_cf_connecting_ipv6",' - '"cf_ray":"$${keepempty}cf_ray"' + '"cf_ray":"$${keepempty}http_cf_ray"' '}'; access_log /var/log/nginx/access.log main; From 23be02b155e5bca959794dcf1b9d2bf94ba82c3b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 16 Jul 2024 17:33:05 -0300 Subject: [PATCH 4/7] refactor: repeat less nginx config --- k8s/auth.yaml | 4 +-- k8s/datatracker.yaml | 4 +-- k8s/kustomization.yaml | 2 +- k8s/nginx-auth.conf | 3 +++ k8s/nginx-datatracker.conf | 3 +++ k8s/nginx-logging.conf | 19 ++++++++++++++ k8s/nginx.conf | 52 -------------------------------------- 7 files changed, 30 insertions(+), 57 deletions(-) create mode 100644 k8s/nginx-logging.conf delete mode 100644 k8s/nginx.conf diff --git a/k8s/auth.yaml b/k8s/auth.yaml index ae5509d806..c35cdc8ac2 100644 --- a/k8s/auth.yaml +++ b/k8s/auth.yaml @@ -81,8 +81,8 @@ spec: - name: nginx-tmp mountPath: /tmp - name: dt-cfg - mountPath: /etc/nginx/nginx.conf - subPath: nginx.conf + mountPath: /etc/nginx/conf.d/00logging.conf + subPath: nginx-logging.conf - name: dt-cfg mountPath: /etc/nginx/conf.d/auth.conf subPath: nginx-auth.conf diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 8a9fbb1112..a8a9675687 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -81,8 +81,8 @@ spec: - name: nginx-tmp mountPath: /tmp - name: dt-cfg - mountPath: /etc/nginx/nginx.conf - subPath: nginx.conf + mountPath: /etc/nginx/conf.d/00logging.conf + subPath: nginx-logging.conf - name: dt-cfg mountPath: /etc/nginx/conf.d/datatracker.conf subPath: nginx-datatracker.conf diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index df24143aef..4b79f00753 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -3,7 +3,7 @@ namePrefix: dt- configMapGenerator: - name: files-cfgmap files: - - nginx.conf + - nginx-logging.conf - nginx-auth.conf - nginx-datatracker.conf - settings_local.py diff --git a/k8s/nginx-auth.conf b/k8s/nginx-auth.conf index f3f823dd15..6b855137e7 100644 --- a/k8s/nginx-auth.conf +++ b/k8s/nginx-auth.conf @@ -2,6 +2,9 @@ server { listen 8080 default_server; server_name _; + # Replace default "main" formatter with the ietfjson formatter from nginx-logging.conf + access_log /var/log/nginx/access.log ietfjson; + # Note that regex location matches take priority over non-regex "prefix" matches. Use regexes so that # our deny all rule does not squelch the other locations. location ~ ^/health/nginx$ { diff --git a/k8s/nginx-datatracker.conf b/k8s/nginx-datatracker.conf index d3a73bac9b..ff439fba6a 100644 --- a/k8s/nginx-datatracker.conf +++ b/k8s/nginx-datatracker.conf @@ -2,6 +2,9 @@ server { listen 8080 default_server; server_name _; + # Replace default "main" formatter with the ietfjson formatter from nginx-logging.conf + access_log /var/log/nginx/access.log ietfjson; + location /health/nginx { access_log off; return 200; diff --git a/k8s/nginx-logging.conf b/k8s/nginx-logging.conf new file mode 100644 index 0000000000..a5db253780 --- /dev/null +++ b/k8s/nginx-logging.conf @@ -0,0 +1,19 @@ +# Define JSON log format - must be loaded before config that references it +log_format ietfjson escape=json + '{' + '"time":"$${keepempty}time_iso8601",' + '"remote_ip":"$${keepempty}remote_addr",' + '"request":"$${keepempty}request",' + '"host":"$${keepempty}host",' + '"path":"$${keepempty}request_uri",' + '"method":"$${keepempty}request_method",' + '"status":"$${keepempty}status",' + '"len_bytes":"$${keepempty}body_bytes_sent",' + '"duration_ms":"$${keepempty}request_time",' + '"referer":"$${keepempty}http_referer",' + '"user_agent":"$${keepempty}http_user_agent",' + '"x_forwarded_for":"$${keepempty}http_x_forwarded_for",' + '"cf_connecting_ip":"$${keepempty}http_cf_connecting_ip",' + '"cf_connecting_ipv6":"$${keepempty}http_cf_connecting_ipv6",' + '"cf_ray":"$${keepempty}http_cf_ray"' + '}'; diff --git a/k8s/nginx.conf b/k8s/nginx.conf deleted file mode 100644 index 00cdbbdf4e..0000000000 --- a/k8s/nginx.conf +++ /dev/null @@ -1,52 +0,0 @@ -# nginx configuration -worker_processes auto; - -error_log /var/log/nginx/error.log notice; -pid /tmp/nginx.pid; - - -events { - worker_connections 1024; -} - - -http { - proxy_temp_path /tmp/proxy_temp; - client_body_temp_path /tmp/client_temp; - fastcgi_temp_path /tmp/fastcgi_temp; - uwsgi_temp_path /tmp/uwsgi_temp; - scgi_temp_path /tmp/scgi_temp; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main escape=json - '{' - '"time":"$${keepempty}time_iso8601",' - '"remote_ip":"$${keepempty}remote_addr",' - '"request":"$${keepempty}request",' - '"host":"$${keepempty}host",' - '"path":"$${keepempty}request_uri",' - '"method":"$${keepempty}request_method",' - '"status":"$${keepempty}status",' - '"len_bytes":"$${keepempty}body_bytes_sent",' - '"duration_ms":"$${keepempty}request_time",' - '"referer":"$${keepempty}http_referer",' - '"user_agent":"$${keepempty}http_user_agent",' - '"x_forwarded_for":"$${keepempty}http_x_forwarded_for",' - '"cf_connecting_ip":"$${keepempty}http_cf_connecting_ip",' - '"cf_connecting_ipv6":"$${keepempty}http_cf_connecting_ipv6",' - '"cf_ray":"$${keepempty}http_cf_ray"' - '}'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - #tcp_nopush on; - - keepalive_timeout 65; - - #gzip on; - - include /etc/nginx/conf.d/*.conf; -} From 90c3fff037387693ebb4afd0367ee82add69fe41 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 16 Jul 2024 17:34:10 -0300 Subject: [PATCH 5/7] fix: log more req headers from gunicorn --- ietf/utils/jsonlogger.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ietf/utils/jsonlogger.py b/ietf/utils/jsonlogger.py index a9eeb02ba9..a3451c6a4f 100644 --- a/ietf/utils/jsonlogger.py +++ b/ietf/utils/jsonlogger.py @@ -24,3 +24,10 @@ def add_fields(self, log_record, record, message_dict): log_record.setdefault("user_agent", record.args["a"]) log_record.setdefault("len_bytes", record.args["B"]) log_record.setdefault("duration_ms", record.args["M"]) + log_record.setdefault("host", record.args["{host}i"]) + log_record.setdefault("x_request_start", record.args["{x-request-start}i"]) + log_record.setdefault("x_real_ip", record.args["{x-real-ip}i"]) + log_record.setdefault("x_forwarded_for", record.args["{x-forwarded-for}i"]) + log_record.setdefault("cf_connecting_ip", record.args["{cf-connecting-ip}i"]) + log_record.setdefault("cf_connecting_ipv6", record.args["{cf-connecting-ipv6}i"]) + log_record.setdefault("cf_ray", record.args["{cf-ray}i"]) From b5b43d901b552f82efca757ddc8c43b73fe29a26 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 16 Jul 2024 17:55:53 -0300 Subject: [PATCH 6/7] fix: redirect auth->datatracker, not deny --- k8s/nginx-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/nginx-auth.conf b/k8s/nginx-auth.conf index 6b855137e7..6dd5d6ed56 100644 --- a/k8s/nginx-auth.conf +++ b/k8s/nginx-auth.conf @@ -23,7 +23,7 @@ server { # n.b. (?!...) is a negative lookahead group location ~ ^(/(?!(api/openid/|accounts/login/|accounts/logout/|accounts/reset/|person/.*/photo|group/groupmenu.json)).*) { - deny all; + return 302 https://datatracker.ietf.org$${keepempty}request_uri; } location / { From 852ac8eac65f7d54cd2765340adecec3b1f56e9b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 29 Jul 2024 17:42:08 -0300 Subject: [PATCH 7/7] feat: log X-Forwarded-Proto --- ietf/utils/jsonlogger.py | 1 + k8s/nginx-logging.conf | 1 + 2 files changed, 2 insertions(+) diff --git a/ietf/utils/jsonlogger.py b/ietf/utils/jsonlogger.py index a3451c6a4f..9c7949fd58 100644 --- a/ietf/utils/jsonlogger.py +++ b/ietf/utils/jsonlogger.py @@ -28,6 +28,7 @@ def add_fields(self, log_record, record, message_dict): log_record.setdefault("x_request_start", record.args["{x-request-start}i"]) log_record.setdefault("x_real_ip", record.args["{x-real-ip}i"]) log_record.setdefault("x_forwarded_for", record.args["{x-forwarded-for}i"]) + log_record.setdefault("x_forwarded_proto", record.args["{x-forwarded-proto}i"]) log_record.setdefault("cf_connecting_ip", record.args["{cf-connecting-ip}i"]) log_record.setdefault("cf_connecting_ipv6", record.args["{cf-connecting-ipv6}i"]) log_record.setdefault("cf_ray", record.args["{cf-ray}i"]) diff --git a/k8s/nginx-logging.conf b/k8s/nginx-logging.conf index a5db253780..0938b0530e 100644 --- a/k8s/nginx-logging.conf +++ b/k8s/nginx-logging.conf @@ -13,6 +13,7 @@ log_format ietfjson escape=json '"referer":"$${keepempty}http_referer",' '"user_agent":"$${keepempty}http_user_agent",' '"x_forwarded_for":"$${keepempty}http_x_forwarded_for",' + '"x_forwarded_proto":"$${keepempty}http_x_forwarded_proto",' '"cf_connecting_ip":"$${keepempty}http_cf_connecting_ip",' '"cf_connecting_ipv6":"$${keepempty}http_cf_connecting_ipv6",' '"cf_ray":"$${keepempty}http_cf_ray"'