Added a middleware exception handler for the case of people submitting unicode outside the Basic Multilingual Plane, which cannot currently be saved to the database.

 - Legacy-Id: 13508

Author: levkowetz

ietf/middleware.py

@@ -1,9 +1,10 @@
 # Copyright The IETF Trust 2007, All Rights Reserved
 
 from django.db import connection
+from django.db.utils import OperationalError
 from django.shortcuts import render
 from django.http import HttpResponsePermanentRedirect
-from ietf.utils.log import log
+from ietf.utils.log import log, exception_components
 from ietf.utils.mail import log_smtp_exception
 import re
 import smtplib
@@ -31,6 +32,20 @@ def process_exception(self, request, exception):
                           {'exception': extype, 'args': value, 'traceback': "".join(tb)} )
 	return None
 
+class Utf8ExceptionMiddleware(object):
+    def __init__(self, get_response):
+        self.get_response = get_response
+    def __call__(self, request):
+        return self.get_response(request)
+    def process_exception(self, request, exception):
+	if isinstance(exception, OperationalError):
+            extype, value, tb = exception_components(exception)
+            if value[0] == 1366:
+                log("Database 4-byte utf8 exception: %s: %s" % (extype, value))
+                return render(request, 'utf8_4byte_failed.html',
+                              {'exception': extype, 'args': value, 'traceback': "".join(tb)} )
+	return None
+
 def redirect_trailing_period_middleware(get_response):
     def redirect_trailing_period(request):
         response = get_response(request)
@@ -53,4 +68,4 @@ def unicode_nfkc_normalization(request):
         response = get_response(request)
         return response
     return unicode_nfkc_normalization
-        
+        

ietf/settings.py

@@ -341,6 +341,7 @@ def skip_unreadable_post(record):
     'django.middleware.http.ConditionalGetMiddleware',
     'ietf.middleware.sql_log_middleware',
     'ietf.middleware.SMTPExceptionMiddleware',
+    'ietf.middleware.Utf8ExceptionMiddleware',
     'ietf.middleware.redirect_trailing_period_middleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'ietf.middleware.unicode_nfkc_normalization_middleware',

ietf/templates/utf8_4byte_failed.html

@@ -0,0 +1,21 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{% load origin %}
+
+{% block title %}Unicode outside the Basic Multilingual Plane{% endblock %}
+
+{% block content %}
+  {% origin %}
+  <h1>Unicode outside the Basic Multilingual Plane</h1>
+
+  <p class="alert alert-warning">
+
+    You submitted data containing unicode characters outside the Basic
+    Multilingual Plane.  Such character cannot be stored in the database at
+    this time.  If this causes inconvenience, please send an e-mail with
+    details to <a href="mailto:webmaster@ietf.org">webmaster@ietf.org</a>, in
+    order for us to understand the issue.
+    
+  </p>
+
+{% endblock %}

ietf/utils/log.py

@@ -1,8 +1,10 @@
 # Copyright The IETF Trust 2007, All Rights Reserved
 
+import sys
 import logging
 import inspect
 import os.path
+import traceback
 
 try:
     import syslog
@@ -49,9 +51,15 @@ def log(msg):
         file, line, where = "/<UNKNOWN>", 0, ""
     logfunc("ietf%s(%d)%s: %s" % (file, line, where, msg))
 
+logger = logging.getLogger('django')
 
 
-logger = logging.getLogger('django')
+
+def exception_components(e):
+    extype = sys.exc_info()[0]
+    value = sys.exc_info()[1]
+    tb = traceback.format_tb(sys.exc_info()[2])
+    return (extype, value, tb)
 
 def build_traceback(stack):
     """