feat: use gunicorn (ietf-tools#7215)

* feat: use gunicorn

* fix: let gunicorn emit logs to stdout/stderr

* fix: log to stdout/stderr in json format

* fix: run collectstatic for the local copy of the statics

Author: rjsparks

dev/build/datatracker-start.sh

@@ -6,5 +6,19 @@ echo "Running Datatracker checks..."
 echo "Running Datatracker migrations..."
 ./ietf/manage.py migrate --settings=settings_local
 
+echo "Running collectstatic..."
+./ietf/manage.py collectstatic
+
 echo "Starting Datatracker..."
-./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local
+
+gunicorn \
+          --workers 53 \
+          --max-requests 32768 \
+          --timeout 180 \
+          --bind :8000 \
+          --log-level info \
+          ietf.wsgi:application
+          
+          # Leaving this here as a reminder to set up the env in the chart
+          # Remove this once that's complete.
+          #--env SCOUT_NAME=Datatracker \

ietf/settings.py

@@ -236,7 +236,7 @@ def skip_unreadable_post(record):
     #
     'loggers': {
         'django': {
-            'handlers': ['debug_console', 'mail_admins'],
+            'handlers': ['console', 'mail_admins',],
             'level': 'INFO',
         },
         'django.request': {
@@ -248,13 +248,17 @@ def skip_unreadable_post(record):
             'level': 'INFO',
         },
         'django.security': {
-	    'handlers': ['debug_console', ],
+	        'handlers': ['console', ],
+            'level': 'INFO',
+        },
+ 	    'oidc_provider': {
+	        'handlers': ['debug_console', ],
+	        'level': 'DEBUG',
+	    },
+        'datatracker': {
+            'handlers': ['console', ],
             'level': 'INFO',
         },
- 	'oidc_provider': {
-	    'handlers': ['debug_console', ],
-	    'level': 'DEBUG',
-	},
     },
     #
     # No logger filters
@@ -263,14 +267,7 @@ def skip_unreadable_post(record):
         'console': {
             'level': 'DEBUG',
             'class': 'logging.StreamHandler',
-            'formatter': 'plain',
-        },
-        'syslog': {
-            'level': 'DEBUG',
-            'class': 'logging.handlers.SysLogHandler',
-            'facility': 'user',
-            'formatter': 'plain',
-            'address': '/dev/log',
+            'formatter': 'json',
         },
         'debug_console': {
             # Active only when DEBUG=True
@@ -325,6 +322,9 @@ def skip_unreadable_post(record):
             'style': '{',
             'format': '{levelname}: {name}:{lineno}: {message}',
         },
+        'json' : {
+            '()': 'pythonjsonlogger.jsonlogger.JsonFormatter'
+        }
     },
 }
 

ietf/utils/log.py

@@ -10,30 +10,19 @@
 import traceback
 
 from typing import Callable         # pyflakes:ignore
-
-try:
-    import syslog
-    logfunc = syslog.syslog             # type: Callable
-except ImportError:                     # import syslog will fail on Windows boxes
-    logging.basicConfig(filename='tracker.log',level=logging.INFO)
-    logfunc = logging.info
-    pass
-
 from django.conf import settings
+from pythonjsonlogger import jsonlogger
 
 import debug                            # pyflakes:ignore
 
-formatter = logging.Formatter('{levelname}: {name}:{lineno}: {message}', style='{')
+formatter = jsonlogger.JsonFormatter
 for name, level in settings.UTILS_LOGGER_LEVELS.items():
     logger = logging.getLogger(name)
     if not logger.hasHandlers():
         debug.say(' Adding handlers to logger %s' % logger.name)
-        
         handlers = [
-                logging.StreamHandler(),
-                logging.handlers.SysLogHandler(address='/dev/log',
-                    facility=logging.handlers.SysLogHandler.LOG_USER),
-            ]
+            logging.StreamHandler(),
+        ]
         for h in handlers:
             h.setFormatter(formatter)
             h.setLevel(level)
@@ -56,20 +45,9 @@ def getcaller():
     return (pmodule, pclass, pfunction, pfile, pline)
 
 def log(msg, e=None):
-    "Uses syslog by preference.  Logs the given calling point and message."
-    global logfunc
-    def _flushfunc():
-        pass
-    _logfunc = logfunc
-    if settings.SERVER_MODE == 'test':
-        if getattr(settings, 'show_logging', False) is True:
-            _logfunc = debug.say
-            _flushfunc = sys.stdout.flush   # pyflakes:ignore (intentional redefinition)
-        else:
+    "Logs the given calling point and message to the logging framework's datatracker handler at severity INFO"
+    if settings.SERVER_MODE == 'test' and not getattr(settings, 'show_logging',False):
             return
-    elif settings.DEBUG == True:
-        _logfunc = debug.say
-        _flushfunc = sys.stdout.flush   # pyflakes:ignore (intentional redefinition)
     if not isinstance(msg, str):
         msg = msg.encode('unicode_escape')
     try:
@@ -82,11 +60,8 @@ def _flushfunc():
             where = " in " + func + "()"
     except IndexError:
         file, line, where = "/<UNKNOWN>", 0, ""
-    _flushfunc()
-    _logfunc("ietf%s(%d)%s: %s" % (file, line, where, msg))
-
-logger = logging.getLogger('django')
 
+    logging.getLogger("datatracker").info(msg=msg, extra = {"file":file, "line":line, "where":where})
 
 
 def exc_parts():
@@ -124,6 +99,7 @@ def assertion(statement, state=True, note=None):
     This acts like an assertion.  It uses the django logger in order to send
     the failed assertion and a backtrace as for an internal server error.
     """
+    logger = logging.getLogger("django") # Note this is a change - before this would have gone to "django"
     frame = inspect.currentframe().f_back
     value = eval(statement, frame.f_globals, frame.f_locals)
     if bool(value) != bool(state):
@@ -148,6 +124,7 @@ def assertion(statement, state=True, note=None):
 
 def unreachable(date="(unknown)"):
     "Raises an assertion or sends traceback to admins if executed."
+    logger = logging.getLogger("django")
     frame = inspect.currentframe().f_back
     if settings.DEBUG is True or settings.SERVER_MODE == 'test':
         raise AssertionError("Arrived at code in %s() which was marked unreachable on %s." % (frame.f_code.co_name, date))

requirements.txt

@@ -53,6 +53,7 @@ pyopenssl>=22.0.0    # Used by urllib3.contrib, which is used by PyQuery but not
 pyquery>=1.4.3
 python-dateutil>=2.8.2
 types-python-dateutil>=2.8.2
+python-json-logger>=2.0.7
 python-magic==0.4.18    # Versions beyond the yanked .19 and .20 introduce form failures
 pymemcache>=4.0.0  # for django.core.cache.backends.memcached.PyMemcacheCache
 python-mimeparse>=1.6    # from TastyPie