Merged [7673] from rjsparks@nostrum.com: Made several changes to better handle non-ascii UTF-8 in email messages.

Used multipart mime when including the original message in an error message to the secretariat.
Modified the way IANA review email is parsed.
Added non-ascii UTF-8 tests to the IANA review email parser and to the SMTP Exception handling code.
 - Legacy-Id: 7676
Note: SVN reference [7673] has been migrated to Git commit bf4f3ad

Author: levkowetz

ietf/sync/iana.py

@@ -216,7 +216,7 @@ def update_history_with_changes(changes, send_email=True):
 
 
 def parse_review_email(text):
-    msg = email.message_from_string(text.encode("utf-8"))
+    msg = email.message_from_string(text)
 
     # doc
     doc_name = ""

ietf/sync/tests.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 import os
 import json
 import datetime
@@ -130,7 +131,7 @@ def test_changes_sync_errors(self):
     def test_iana_review_mail(self):
         draft = make_test_data()
 
-        msg = """From: "%(person)s via RT" <drafts-lastcall@iana.org>
+        msg = u"""From: "%(person)s via RT" <drafts-lastcall@iana.org>
 Date: Thu, 10 May 2012 12:00:00 +0000
 Subject: [IANA #12345] Last Call: <%(draft)s-%(rev)s.txt> (Long text) to Informational RFC
 
@@ -147,7 +148,7 @@ def test_iana_review_mail(self):
 Thanks,
 
 %(person)s
-IANA Fake Test Person
+IANA “Fake Test” Person
 ICANN
 
 (END IANA LAST CALL COMMENTS)
@@ -156,8 +157,8 @@ def test_iana_review_mail(self):
         msg = msg % dict(person=Person.objects.get(user__username="iana").name,
                          draft=draft.name,
                          rev=draft.rev)
-
-        doc_name, review_time, by, comment = iana.parse_review_email(msg)
+ 
+        doc_name, review_time, by, comment = iana.parse_review_email(msg.encode('utf-8'))
 
         self.assertEqual(doc_name, draft.name)
 #        self.assertEqual(review_time, datetime.datetime(2012, 5, 10, 5, 0, 0))

ietf/utils/mail.py

@@ -304,17 +304,20 @@ def smtp_error_user_warning(thing,request):
     except smtplib.SMTPException as e:
         (extype, value, tb) = log_smtp_exception(e)
 
-        warning =  "An error occured while sending email with\n"
-        warning += "Subject: %s\n" % e.original_msg.get('Subject','[no subject]')
-        warning += "To: %s\n" % e.original_msg.get('To','[no to]')
-        warning += "Cc: %s\n" % e.original_msg.get('Cc','[no cc]')
-        if isinstance(e,SMTPSomeRefusedRecipients):
-            warning += e.detailed_refusals()
-        else:
-            warning += "SMTP Exception: %s\n"%extype
-            warning += "Error Message: %s\n\n"%value
-            warning += "The message was not delivered to anyone."
-        messages.warning(request,warning,extra_tags='preformatted',fail_silently=True)
+        if request:
+            warning =  "An error occured while sending email:\n"
+            if (e.original_msg):
+                warning += "Subject: %s\n" % e.original_msg.get('Subject','[no subject]')
+                warning += "To: %s\n" % e.original_msg.get('To','[no to]')
+                warning += "Cc: %s\n" % e.original_msg.get('Cc','[no cc]')
+            if isinstance(e,SMTPSomeRefusedRecipients):
+                warning += e.detailed_refusals()
+            else:
+                warning += "SMTP Exception: %s\n"%extype
+                warning += "Error Message: %s\n\n"%value
+                warning += "The message was not delivered to anyone."
+            messages.warning(request,warning,extra_tags='preformatted',fail_silently=True)
+
         raise
 
 @contextmanager
@@ -324,68 +327,52 @@ def smtp_error_logging(thing):
     except smtplib.SMTPException as e:
         (extype, value, tb) = log_smtp_exception(e)
 
-        msg = textwrap.dedent("""\
-                  To: <action@ietf.org> 
-                  From: %s
-                  """) % settings.SERVER_EMAIL
+        msg = MIMEMultipart()
+        msg['To'] = '<action@ietf.org>'
+        msg['From'] = settings.SERVER_EMAIL
         if isinstance(e,SMTPSomeRefusedRecipients):
-            msg += textwrap.dedent("""\
-                      Subject: Some recipients were refused while sending mail with Subject: %s
+            msg['Subject'] = 'Subject: Some recipients were refused while sending mail with Subject: %s' % e.original_msg.get('Subject','[no subject]')
+            textpart = textwrap.dedent("""\
+                This is a message from the datatracker to IETF-Action about an email
+                delivery failure, when sending email from the datatracker.
 
-                      This is a message from the datatracker to IETF-Action about an email
-                      delivery failure, when sending email from the datatracker.
+                %s
 
-                      %s
-
-                      The original message follows:
-                      -------- BEGIN ORIGINAL MESSAGE --------
-                      %s
-                      --------- END ORIGINAL MESSAGE ---------
-                      """) % (e.original_msg.get('Subject', '[no subject]'),e.detailed_refusals(),e.original_msg.as_string())
+                """) % e.detailed_refusals()
         else:
-            msg += textwrap.dedent("""\
-                      Subject: Datatracker error while sending email
-
-                      This is a message from the datatracker to IETF-Action about an email
-                      delivery failure, when sending email from the datatracker.
+            msg['Subject'] = 'Datatracker error while sending email'
+            textpart = textwrap.dedent("""\
+                This is a message from the datatracker to IETF-Action about an email
+                delivery failure, when sending email from the datatracker.
 
-                      The original message was not delivered to anyone.
+                The original message was not delivered to anyone.
 
-                      SMTP Exception: %s
+                SMTP Exception: %s
 
-                      Error Message: %s
+                Error Message: %s
                      
-                      """) % (extype,value)
-            if hasattr(e,'original_msg'):
-                msg += textwrap.dedent("""\
-                      The original message follows:
-                      -------- BEGIN ORIGINAL MESSAGE --------
-                      %s
-                      --------- END ORIGINAL MESSAGE ---------
-                      """) % e.original_msg.as_string()
+                """) % (extype,value)
+        if hasattr(e,'original_msg'):
+            textpart += "The original message follows:\n"
+        msg.attach(MIMEText(textpart,_charset='utf-8'))
+        if hasattr(e,'original_msg'):
+            msg.attach(MIMEMessage(e.original_msg))
 
         send_error_to_secretariat(msg)
 
-def send_error_to_secretariat(raw_msg):
-
-    (parsed_msg , headers , bcc) = parse_preformatted(raw_msg)
-    send_msg = encode_message(parsed_msg.get_payload())
-    cc = None
-    condition_message(parsed_msg['To'], parsed_msg['From'], parsed_msg['Subject'], send_msg, cc, headers)
+def send_error_to_secretariat(msg):
 
     debugging = getattr(settings, "USING_DEBUG_EMAIL_SERVER", False) and settings.EMAIL_HOST == 'localhost' and settings.EMAIL_PORT == 2025
 
     try:
         if test_mode or debugging or settings.SERVER_MODE == 'production':
-            send_smtp(send_msg, bcc)
+            send_smtp(msg, bcc=None)
         try:
             copy_to = settings.EMAIL_COPY_TO
         except AttributeError:
             copy_to = "ietf.tracker.archive+%s@gmail.com" % settings.SERVER_MODE
         if copy_to and not test_mode and not debugging: # if we're running automated tests, this copy is just annoying
-            if bcc:
-                send_msg['X-Tracker-Bcc']=bcc
-            copy_email(send_msg, copy_to,originalBcc=bcc)
+            copy_email(msg, copy_to,originalBcc=None)
     except smtplib.SMTPException:
         log("Exception encountered while sending a ticket to the secretariat")
         (extype,value) = sys.exc_info()[:2]

ietf/utils/tests.py

@@ -1,10 +1,16 @@
+# -*- coding: utf-8 -*-
 import os.path
 
+from textwrap import dedent
+from email.mime.text import MIMEText
+from email.mime.image import MIMEImage
+from email.mime.multipart import MIMEMultipart
+
 from django.conf import settings
 from django.test import TestCase
 
 from ietf.utils.management.commands import pyflakes
-from ietf.utils.mail import send_mail_text, outbox 
+from ietf.utils.mail import send_mail_text, send_mail_mime, outbox 
 
 class PyFlakesTestCase(TestCase):
 
@@ -18,16 +24,37 @@ class TestSMTPServer(TestCase):
 
     def test_address_rejected(self):
 
-        def send_mail(to):
-           send_mail_text(None, to=to, frm=None, subject="Test for rejection", txt="dummy body")
+        def send_simple_mail(to):
+            send_mail_text(None, to=to, frm=None, subject="Test for rejection", txt="dummy body")
 
         len_before = len(outbox)
-        send_mail('good@example.com,poison@example.com')
+        send_simple_mail('good@example.com,poison@example.com')
         self.assertEqual(len(outbox),len_before+2)
         self.assertTrue('Some recipients were refused' in outbox[-1]['Subject'])
 
         len_before = len(outbox)
-        send_mail('poison@example.com')
+        send_simple_mail('poison@example.com')
         self.assertEqual(len(outbox),len_before+2)
         self.assertTrue('error while sending email' in outbox[-1]['Subject'])
 
+    def test_rejecting_complex_mail(self):
+
+        def send_complex_mail(to):
+            msg = MIMEMultipart()
+            textpart= MIMEText(dedent(u"""\
+                             Sometimes people send mail with things like “smart quotes” in them.
+                             Sometimes they have attachments with pictures.
+                              """),_charset='utf-8')
+            msg.attach(textpart)
+            img = MIMEImage('\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x06\x00\x00\x00\x1f\xf3\xffa\x00\x00\x02\x88IDATx\xda\xa5\x93\xcbO\x13Q\x14\xc6\xbf\xb9\xd32}L\x9fZ\x06\x10Q\x90\x10\x85b\x89\xfe\x01\x06BXK"\xd4hB\xdc\xa0\x06q\xe1c% H1l\xd0\x8dbT6\x1a5\x91\x12#K\x891\xf2\x07\x98\xc8[L\x1ay\xa8@\xdb\xd0\xd2\xe9\x83N;\xbdc\x1f\x11\x03\x04\x17zW\'_\xce\xf9\xdd\xef\x9c\x9c\xc3\xe0?\x1f\xb3S\xf8\xfe\xba\xc2Be\xa9]m\xd6\x9e\xe6x\xde\x9e\xd1\xa4HdF\x0e\xc5G\x89\x8a{X\xec\xfc\x1a\xdc\x1307X\xd4$T\nC\xc6\xfc|\x13\x8d\xa6\x00\xe5O\x16\xd1\xb3\x10}\xbe\x90w\xce\xdbZyeed\x17\xc03(4\x15\x9d(s\x13\xca!\x10\xa6\xb0\x1a\xb6\x9b\x0b\x84\x95\xb4F@\x89\x84\xe5O\x0b\xcdG\xaf\xae\x8dl\x01V\x9f\x1d\xb4q\x16\xde\xa33[\x8d\xe3\x93)\xdc\x7f\x9b\xc4\xf3\x1b\x1c,|\xaex#\n\xb4\x0cH\xb8\xd6\xa8F\xad\x83El# \xc6\x83\xb1\xf2\xa2\x0bK\xfe,`y\xd0\xe6*<V\xda\x99\x92\x15\xb8\xdc\x14\xef>\x03\xaes\x0c\xea\xaas.\xc6g\x14t\xbcR\xd0P\x03t;\tX\x15\x83\xd5\xf9\xc5\xbe\x926_W6\xe3\xe7ca\xc2Z,82\xb1\xeb\r\x8b\xb1)\x82\xde\xa6\x14\xea\xec4\x0b\xf88K\xd0\xe5fQ_M\xd1s&\x95k\xe9\x87w\xf2\xc0eoM\x16\xe0\x1b*\xdc4XM\x9aL\xfca\x8e\xc5\xbd1\x0e//\xc6`\xd5\xe7Z\x08\xa6[8\xffT\x87\xeb\r\x12\xea\xabr\x80p \x14\xcfo]\xd5f\x01k\x8fl\x9bF3\xaf\xf9=\xb0X\x82\x81.O\xd96\xc4\x9d\x9a\xb8\x11\x89\x17\xb4\xf9s\x80\xe5\x01\xc3\xc4\xfe}FG\\\x064\xaa\xbf/\x0eM3\x92i\x13\xe1\x908Yr3\x9ck\xe1[\xbf\xd6%X\xf4\x9d\xef=z$(\xc1\xa9\xc3Q\xf0\x1c\xddV(\xa7\x18Ly9L\xafq8{\\D0\x14\xbd{\xe4V\xac3\x0bX\xe8\xd7\xdb\xb4,\xf5\x18\xb4j\xe3\xf8\xa2\x1e/\xa6\xac`\x18\x06\x02\x9f\x84\x8a\xa4\x07\x16c\xb1\xbe\xc9\xa2\xf6P\x04-\x8e\x00\x12\xc9\x84(&\xd9\xf2\x8a\x8e\x88\x7fk[\xbet\xe75\x0bzf\x98cI\xd6\xe6\xfc\xba\x06\xd3~\x1d\x12\xe9\x9fK\xcd\x12N\x16\xc4\xa0UQH)\x8a\x95\x08\x9c\xf6^\xc9\xbdk\x95\xe7o\xab\x9c&\xb5\xf2\x84W3\xa6\x9dG\x92\x19_$\xa9\x84\xd6%r\xc9\xde\x97\x1c\xde\xf3\x98\x96\xee\xb0\x16\x99\xd2v\x15\x94\xc6<\xc2Te\xb4\x04Ufe\x85\x8c2\x84<(\xeb\x91\xf7>\xa6\x7fy\xbf\x00\x96T\xff\x11\xf7\xd8R\xb9\x00\x00\x00\x00IEND\xaeB`\x82')
+
+            msg.attach(img)
+            send_mail_mime(request=None, to=to, frm=settings.DEFAULT_FROM_EMAIL, subject=u'это сложно', msg=msg, cc=None, extra=None)
+
+        len_before = len(outbox)
+        send_complex_mail('good@example.com')
+        self.assertEqual(len(outbox),len_before+1)
+
+        len_before = len(outbox)
+        send_complex_mail('good@example.com,poison@example.com')
+        self.assertEqual(len(outbox),len_before+2)