From c0fcbad9ac3fc756b2fee414c559d77a014bd024 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 9 Jan 2025 08:56:14 -0600 Subject: [PATCH 1/3] feat: additional filesystem monitoring --- dev/deploy-to-container/settings_local.py | 1 + dev/diff/settings_local.py | 1 + dev/tests/settings_local.py | 1 + docker/configs/settings_local.py | 1 + docker/scripts/app-create-dirs.sh | 1 + ietf/api/tests.py | 8 ++++++++ ietf/api/urls.py | 2 ++ ietf/api/views.py | 20 +++++++++++++++++++- ietf/settings.py | 1 + 9 files changed, 35 insertions(+), 1 deletion(-) diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index 07bf0a75113..4c2ff5b076a 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -64,6 +64,7 @@ BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml' IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH FTP_DIR = '/assets/ftp' +TMP_DIR = '/assets/tmp' NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/' SLIDE_STAGING_PATH = '/test/staging/' diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py index 6bcee46b61f..f0da2dbd78e 100644 --- a/dev/diff/settings_local.py +++ b/dev/diff/settings_local.py @@ -60,6 +60,7 @@ INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/' BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml' FTP_DIR = '/assets/ftp' +TMP_DIR = '/assets/tmp' NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/' SLIDE_STAGING_PATH = 'test/staging/' diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index afadb3760bf..514f043afe2 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -59,6 +59,7 @@ INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/' BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml' FTP_DIR = '/assets/ftp' +TMP_DIR = '/assets/tmp' NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/' SLIDE_STAGING_PATH = 'test/staging/' diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index a1c19c80cfa..1e5e58fe15b 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -50,6 +50,7 @@ BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml' IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH FTP_DIR = '/assets/ftp' +TMP_DIR = '/assets/tmp' NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/' SLIDE_STAGING_PATH = '/assets/www6s/staging/' diff --git a/docker/scripts/app-create-dirs.sh b/docker/scripts/app-create-dirs.sh index 50431f47938..3eb328a2807 100755 --- a/docker/scripts/app-create-dirs.sh +++ b/docker/scripts/app-create-dirs.sh @@ -29,6 +29,7 @@ for sub in \ /assets/www6/iesg \ /assets/www6/iesg/evaluation \ /assets/media/photo \ + /assets/tmp \ /assets/ftp \ /assets/ftp/charter \ /assets/ftp/internet-drafts \ diff --git a/ietf/api/tests.py b/ietf/api/tests.py index a8d6ac4e577..6e7fb374fe7 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -970,6 +970,14 @@ def test_api_appauth(self): self.assertEqual(jsondata['success'], True) self.client.logout() + @override_settings(APP_API_TOKENS={"ietf.api.views.nfs_metrics": ["valid-token"]}) + def test_api_nfs_metrics(self): + url = urlreverse("ietf.api.views.nfs_metrics") + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, 'nfs_latency_seconds{operation="write"}') + def test_api_get_session_matherials_no_agenda_meeting_url(self): meeting = MeetingFactory(type_id='ietf') session = SessionFactory(meeting=meeting) diff --git a/ietf/api/urls.py b/ietf/api/urls.py index a9aaaf58054..a9dd5999c40 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -82,6 +82,8 @@ url(r'^version/?$', api_views.version), # Application authentication API key url(r'^appauth/(?Pauthortools|bibxml)$', api_views.app_auth), + # NFS metrics endpoint + url(r'^nfs_metrics/?$', api_views.nfs_metrics), # latest versions url(r'^rfcdiff-latest-json/%(name)s(?:-%(rev)s)?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, api_views.rfcdiff_latest_json), url(r'^rfcdiff-latest-json/(?P[Rr][Ff][Cc] [0-9]+?)(\.txt|\.html)?/?$', api_views.rfcdiff_latest_json), diff --git a/ietf/api/views.py b/ietf/api/views.py index 3e56757528d..fb9fc2c827c 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -3,7 +3,10 @@ import base64 import binascii +import datetime import json +from pathlib import Path +from tempfile import NamedTemporaryFile import jsonschema import pytz import re @@ -264,7 +267,22 @@ def app_auth(request, app: Literal["authortools", "bibxml"]): json.dumps({'success': True}), content_type='application/json') - +@requires_api_token +@csrf_exempt +def nfs_metrics(request): + with NamedTemporaryFile(dir=settings.TMP_DIR,delete=False) as fp: + fp.close() + mark = datetime.datetime.now() + with open(fp.name, mode="w") as f: + f.write("whyioughta"*1024) + write_latency = (datetime.datetime.now() - mark).total_seconds() + mark = datetime.datetime.now() + with open(fp.name, "r") as f: + _=f.read() + read_latency = (datetime.datetime.now() - mark).total_seconds() + Path(f.name).unlink() + response=f'nfs_latency_seconds{{operation="write"}} {write_latency}\nnfs_latency_seconds{{operation="read"}} {read_latency}\n' + return HttpResponse(response) def find_doc_for_rfcdiff(name, rev): """rfcdiff lookup heuristics diff --git a/ietf/settings.py b/ietf/settings.py index efd04c6068b..17a2f440a7a 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -761,6 +761,7 @@ def skip_unreadable_post(record): DERIVED_DIR = '/a/ietfdata/derived' FTP_DIR = '/a/ftp' ALL_ID_DOWNLOAD_DIR = '/a/www/www6s/download' +TMP_DIR = '/a/tmp' DOCUMENT_FORMAT_ALLOWLIST = ["txt", "ps", "pdf", "xml", "html", ] From 247265f0792cf8fa01c2d1d537570a51ee64cd7d Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 9 Jan 2025 11:13:54 -0600 Subject: [PATCH 2/3] chore: rename setting for tmp directory --- dev/deploy-to-container/settings_local.py | 2 +- dev/diff/settings_local.py | 2 +- dev/tests/settings_local.py | 2 +- docker/configs/settings_local.py | 2 +- ietf/api/views.py | 2 +- ietf/settings.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index 4c2ff5b076a..0a991ae9fe2 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -64,7 +64,7 @@ BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml' IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH FTP_DIR = '/assets/ftp' -TMP_DIR = '/assets/tmp' +NFS_METRICS_TMP_DIR = '/assets/tmp' NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/' SLIDE_STAGING_PATH = '/test/staging/' diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py index f0da2dbd78e..95d1e481c9c 100644 --- a/dev/diff/settings_local.py +++ b/dev/diff/settings_local.py @@ -60,7 +60,7 @@ INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/' BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml' FTP_DIR = '/assets/ftp' -TMP_DIR = '/assets/tmp' +NFS_METRICS_TMP_DIR = '/assets/tmp' NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/' SLIDE_STAGING_PATH = 'test/staging/' diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index 514f043afe2..7b10bee06a8 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -59,7 +59,7 @@ INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/' BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml' FTP_DIR = '/assets/ftp' -TMP_DIR = '/assets/tmp' +NFS_METRICS_TMP_DIR = '/assets/tmp' NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/' SLIDE_STAGING_PATH = 'test/staging/' diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 1e5e58fe15b..5df5d15e82b 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -50,7 +50,7 @@ BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml' IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH FTP_DIR = '/assets/ftp' -TMP_DIR = '/assets/tmp' +NFS_METRICS_TMP_DIR = '/assets/tmp' NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/' SLIDE_STAGING_PATH = '/assets/www6s/staging/' diff --git a/ietf/api/views.py b/ietf/api/views.py index fb9fc2c827c..2fd9d2730fa 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -270,7 +270,7 @@ def app_auth(request, app: Literal["authortools", "bibxml"]): @requires_api_token @csrf_exempt def nfs_metrics(request): - with NamedTemporaryFile(dir=settings.TMP_DIR,delete=False) as fp: + with NamedTemporaryFile(dir=settings.NFS_METRICS_TMP_DIR,delete=False) as fp: fp.close() mark = datetime.datetime.now() with open(fp.name, mode="w") as f: diff --git a/ietf/settings.py b/ietf/settings.py index 17a2f440a7a..b452864be67 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -761,7 +761,7 @@ def skip_unreadable_post(record): DERIVED_DIR = '/a/ietfdata/derived' FTP_DIR = '/a/ftp' ALL_ID_DOWNLOAD_DIR = '/a/www/www6s/download' -TMP_DIR = '/a/tmp' +NFS_METRICS_TMP_DIR = '/a/tmp' DOCUMENT_FORMAT_ALLOWLIST = ["txt", "ps", "pdf", "xml", "html", ] From 14d548e0899f46cb9638d45e3e265db9677a63fa Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 9 Jan 2025 11:16:54 -0600 Subject: [PATCH 3/3] fix: restructure path to new endpoint --- ietf/api/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/api/urls.py b/ietf/api/urls.py index a9dd5999c40..b0dbaf91ce7 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -83,7 +83,7 @@ # Application authentication API key url(r'^appauth/(?Pauthortools|bibxml)$', api_views.app_auth), # NFS metrics endpoint - url(r'^nfs_metrics/?$', api_views.nfs_metrics), + url(r'^metrics/nfs/?$', api_views.nfs_metrics), # latest versions url(r'^rfcdiff-latest-json/%(name)s(?:-%(rev)s)?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, api_views.rfcdiff_latest_json), url(r'^rfcdiff-latest-json/(?P[Rr][Ff][Cc] [0-9]+?)(\.txt|\.html)?/?$', api_views.rfcdiff_latest_json),