From a727bcfcdc1b22576687e8253f4b02128e07fee8 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 19 Mar 2024 21:49:00 -0300 Subject: [PATCH 1/5] fix: log to stdout/stderr in json format --- ietf/settings.py | 28 ++++++++++++++-------------- ietf/utils/log.py | 41 +++++++++-------------------------------- requirements.txt | 1 + 3 files changed, 24 insertions(+), 46 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index cd8c0700a1..b80f668f8e 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -236,7 +236,7 @@ def skip_unreadable_post(record): # 'loggers': { 'django': { - 'handlers': ['debug_console', 'mail_admins'], + 'handlers': ['console', 'mail_admins',], 'level': 'INFO', }, 'django.request': { @@ -248,13 +248,17 @@ def skip_unreadable_post(record): 'level': 'INFO', }, 'django.security': { - 'handlers': ['debug_console', ], + 'handlers': ['console', ], + 'level': 'INFO', + }, + 'oidc_provider': { + 'handlers': ['debug_console', ], + 'level': 'DEBUG', + }, + 'datatracker': { + 'handlers': ['console', ], 'level': 'INFO', }, - 'oidc_provider': { - 'handlers': ['debug_console', ], - 'level': 'DEBUG', - }, }, # # No logger filters @@ -263,14 +267,7 @@ def skip_unreadable_post(record): 'console': { 'level': 'DEBUG', 'class': 'logging.StreamHandler', - 'formatter': 'plain', - }, - 'syslog': { - 'level': 'DEBUG', - 'class': 'logging.handlers.SysLogHandler', - 'facility': 'user', - 'formatter': 'plain', - 'address': '/dev/log', + 'formatter': 'json', }, 'debug_console': { # Active only when DEBUG=True @@ -325,6 +322,9 @@ def skip_unreadable_post(record): 'style': '{', 'format': '{levelname}: {name}:{lineno}: {message}', }, + 'json' : { + '()': 'pythonjsonlogger.jsonlogger.JsonFormatter' + } }, } diff --git a/ietf/utils/log.py b/ietf/utils/log.py index d5a54e5516..324e8937f9 100644 --- a/ietf/utils/log.py +++ b/ietf/utils/log.py @@ -10,30 +10,19 @@ import traceback from typing import Callable # pyflakes:ignore - -try: - import syslog - logfunc = syslog.syslog # type: Callable -except ImportError: # import syslog will fail on Windows boxes - logging.basicConfig(filename='tracker.log',level=logging.INFO) - logfunc = logging.info - pass - from django.conf import settings +from pythonjsonlogger import jsonlogger import debug # pyflakes:ignore -formatter = logging.Formatter('{levelname}: {name}:{lineno}: {message}', style='{') +formatter = jsonlogger.JsonFormatter for name, level in settings.UTILS_LOGGER_LEVELS.items(): logger = logging.getLogger(name) if not logger.hasHandlers(): debug.say(' Adding handlers to logger %s' % logger.name) - handlers = [ - logging.StreamHandler(), - logging.handlers.SysLogHandler(address='/dev/log', - facility=logging.handlers.SysLogHandler.LOG_USER), - ] + logging.StreamHandler(), + ] for h in handlers: h.setFormatter(formatter) h.setLevel(level) @@ -56,20 +45,9 @@ def getcaller(): return (pmodule, pclass, pfunction, pfile, pline) def log(msg, e=None): - "Uses syslog by preference. Logs the given calling point and message." - global logfunc - def _flushfunc(): - pass - _logfunc = logfunc - if settings.SERVER_MODE == 'test': - if getattr(settings, 'show_logging', False) is True: - _logfunc = debug.say - _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) - else: + "Logs the given calling point and message to the logging framework's datatracker handler at severity INFO" + if settings.SERVER_MODE == 'test' and not getattr(settings, 'show_logging',False): return - elif settings.DEBUG == True: - _logfunc = debug.say - _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) if not isinstance(msg, str): msg = msg.encode('unicode_escape') try: @@ -82,11 +60,8 @@ def _flushfunc(): where = " in " + func + "()" except IndexError: file, line, where = "/", 0, "" - _flushfunc() - _logfunc("ietf%s(%d)%s: %s" % (file, line, where, msg)) - -logger = logging.getLogger('django') + logging.getLogger("datatracker").info(msg=msg, extra = {"file":file, "line":line, "where":where}) def exc_parts(): @@ -124,6 +99,7 @@ def assertion(statement, state=True, note=None): This acts like an assertion. It uses the django logger in order to send the failed assertion and a backtrace as for an internal server error. """ + logger = logging.getLogger("django") # Note this is a change - before this would have gone to "django" frame = inspect.currentframe().f_back value = eval(statement, frame.f_globals, frame.f_locals) if bool(value) != bool(state): @@ -148,6 +124,7 @@ def assertion(statement, state=True, note=None): def unreachable(date="(unknown)"): "Raises an assertion or sends traceback to admins if executed." + logger = logging.getLogger("django") frame = inspect.currentframe().f_back if settings.DEBUG is True or settings.SERVER_MODE == 'test': raise AssertionError("Arrived at code in %s() which was marked unreachable on %s." % (frame.f_code.co_name, date)) diff --git a/requirements.txt b/requirements.txt index 9231e400a7..36c0f5d4e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,6 +53,7 @@ pyopenssl>=22.0.0 # Used by urllib3.contrib, which is used by PyQuery but not pyquery>=1.4.3 python-dateutil>=2.8.2 types-python-dateutil>=2.8.2 +python-json-logger>=2.0.7 python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form failures pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=1.6 # from TastyPie From d77459082058110c0929f7b0c1b4f8bc4290af74 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 14 May 2024 11:41:44 -0300 Subject: [PATCH 2/5] chore: remove UTILS_LOGGER_LEVELS This is not used (there _is_ a setting for the django.security logger in settings_local.py on production, but it is redundant with the settings.LOGGING configuration and is not doing anything). --- ietf/settings.py | 9 --------- ietf/utils/log.py | 14 -------------- ietf/utils/management/commands/showloggers.py | 13 +------------ 3 files changed, 1 insertion(+), 35 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index b80f668f8e..1fcd699cdb 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -328,15 +328,6 @@ def skip_unreadable_post(record): }, } -# This should be overridden by settings_local for any logger where debug (or -# other) custom log settings are wanted. Use "ietf/manage.py showloggers -l" -# to show registered loggers. The content here should match the levels above -# and is shown as an example: -UTILS_LOGGER_LEVELS: Dict[str, str] = { -# 'django': 'INFO', -# 'django.server': 'INFO', -} - # End logging # ------------------------------------------------------------------------ diff --git a/ietf/utils/log.py b/ietf/utils/log.py index 324e8937f9..b8739d9382 100644 --- a/ietf/utils/log.py +++ b/ietf/utils/log.py @@ -15,20 +15,6 @@ import debug # pyflakes:ignore -formatter = jsonlogger.JsonFormatter -for name, level in settings.UTILS_LOGGER_LEVELS.items(): - logger = logging.getLogger(name) - if not logger.hasHandlers(): - debug.say(' Adding handlers to logger %s' % logger.name) - handlers = [ - logging.StreamHandler(), - ] - for h in handlers: - h.setFormatter(formatter) - h.setLevel(level) - logger.addHandler(h) - debug.say(" Setting %s logging level to %s" % (logger.name, level)) - logger.setLevel(level) def getclass(frame): cls = None diff --git a/ietf/utils/management/commands/showloggers.py b/ietf/utils/management/commands/showloggers.py index 3de9db0c06..b79da9ce26 100644 --- a/ietf/utils/management/commands/showloggers.py +++ b/ietf/utils/management/commands/showloggers.py @@ -11,18 +11,7 @@ import debug # pyflakes:ignore class Command(BaseCommand): - """ - Display a list or tree representation of python loggers. - - Add a UTILS_LOGGER_LEVELS setting in settings_local.py to configure - non-default logging levels for any registered logger, for instance: - - UTILS_LOGGER_LEVELS = { - 'oicd_provider': 'DEBUG', - 'urllib3.connection': 'DEBUG', - } - - """ + """Display a list or tree representation of python loggers""" help = dedent(__doc__).strip() From 8838a15e895e5bd3d80a02ca2b5149935c011b68 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 14 May 2024 12:53:39 -0300 Subject: [PATCH 3/5] chore: revert to debug_console django logging --- ietf/settings.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index 1fcd699cdb..9bba1501b7 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -236,7 +236,7 @@ def skip_unreadable_post(record): # 'loggers': { 'django': { - 'handlers': ['console', 'mail_admins',], + 'handlers': ['debug_console', 'mail_admins',], 'level': 'INFO', }, 'django.request': { @@ -248,7 +248,7 @@ def skip_unreadable_post(record): 'level': 'INFO', }, 'django.security': { - 'handlers': ['console', ], + 'handlers': ['debug_console', ], 'level': 'INFO', }, 'oidc_provider': { @@ -281,6 +281,13 @@ def skip_unreadable_post(record): 'class': 'logging.StreamHandler', 'formatter': 'django.server', }, + 'syslog': { + 'level': 'DEBUG', + 'class': 'logging.handlers.SysLogHandler', + 'facility': 'user', + 'formatter': 'plain', + 'address': '/dev/log', + }, 'mail_admins': { 'level': 'ERROR', 'filters': [ From 4ab7ed42c4a527616ca42063ca16fd3ed1f1824f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 14 May 2024 12:54:03 -0300 Subject: [PATCH 4/5] chore: log.log to syslog via datatracker logger --- ietf/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/settings.py b/ietf/settings.py index 9bba1501b7..534c23d0d7 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -256,7 +256,7 @@ def skip_unreadable_post(record): 'level': 'DEBUG', }, 'datatracker': { - 'handlers': ['console', ], + 'handlers': ['syslog'], 'level': 'INFO', }, }, From 1e23fdc41efaa66ea294359e214a6f4231830996 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 14 May 2024 14:58:36 -0300 Subject: [PATCH 5/5] chore: remove unused imports --- ietf/utils/log.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ietf/utils/log.py b/ietf/utils/log.py index b8739d9382..2a068ade9a 100644 --- a/ietf/utils/log.py +++ b/ietf/utils/log.py @@ -9,9 +9,7 @@ import os.path import traceback -from typing import Callable # pyflakes:ignore from django.conf import settings -from pythonjsonlogger import jsonlogger import debug # pyflakes:ignore