Added explicit charset setting and consistent encoding to several email paths through the datatracker. After introducing support for non-ascii names in email addresses, using role.formatted_email() to insert names into email content doesn't work any more, since uncode names will now be rfc2047-encoded in formatted_email(). Added another method role.name_and_email() for this case. Replaced another case of '%s <%s>' name-and-email formatting with formataddr(). Fixed some tests which did not send unicode email bodies to the email functions. Fixes issue ietf-tools#2017.

 - Legacy-Id: 12896

Author: levkowetz

ietf/doc/mails.py

@@ -369,6 +369,7 @@ def email_iana(request, doc, to, msg, cc=None):
     # fix up message and send it with extra info on doc in headers
     import email
     parsed_msg = email.message_from_string(msg.encode("utf-8"))
+    parsed_msg.set_charset('UTF-8')
 
     extra = {}
     extra["Reply-To"] = "noreply@ietf.org"
@@ -377,7 +378,7 @@ def email_iana(request, doc, to, msg, cc=None):
 
     send_mail_text(request, to,
                    parsed_msg["From"], parsed_msg["Subject"],
-                   parsed_msg.get_payload(),
+                   parsed_msg.get_payload().decode(str(parsed_msg.get_charset())),
                    extra=extra,
                    cc=cc)
 

ietf/doc/utils_charter.py

@@ -5,6 +5,8 @@
 from django.template.loader import render_to_string
 from django.utils.encoding import smart_text
 
+import debug                            # pyflakes:ignore
+
 from ietf.doc.models import NewRevisionDocEvent, WriteupDocEvent 
 from ietf.group.models import ChangeStateGroupEvent
 from ietf.name.models import GroupStateName

ietf/group/models.py

@@ -257,6 +257,13 @@ class Role(models.Model):
     def __unicode__(self):
         return u"%s is %s in %s" % (self.person.plain_name(), self.name.name, self.group.acronym or self.group.name)
 
+    def name_and_email(self):
+        "Returns name and email, e.g.: u'Ano Nymous <ano@nymous.org>' "
+        if self.person:
+            return u"%s <%s>" % (self.person.plain_name(), self.email.address)
+        else:
+            return u"<%s>" % self.address
+
     def formatted_ascii_email(self):
         return email.utils.formataddr((self.person.plain_ascii(), self.email.address))
 

ietf/group/views_edit.py

@@ -324,10 +324,10 @@ def diff(attr, name):
                     added = set(new) - set(old)
                     deleted = set(old) - set(new)
                     if added:
-                        change_text=title + ' added: ' + ", ".join(x.formatted_email() for x in added)
+                        change_text=title + ' added: ' + ", ".join(x.name_and_email() for x in added)
                         personnel_change_text+=change_text+"\n"
                     if deleted:
-                        change_text=title + ' deleted: ' + ", ".join(x.formatted_email() for x in deleted)
+                        change_text=title + ' deleted: ' + ", ".join(x.name_and_email() for x in deleted)
                         personnel_change_text+=change_text+"\n"
                     changed_personnel.update(set(old)^set(new))
 

ietf/mailtrigger/models.py

@@ -216,7 +216,7 @@ def gather_submission_confirmers(self, **kwargs):
                     if doc.stream_id and doc.stream_id not in ['ietf']:
                         addrs.extend(Recipient.objects.get(slug='stream_managers').gather(**{'streams':[doc.stream_id]}))
             else:
-                addrs.extend([u"%s <%s>" % (author["name"], author["email"]) for author in submission.authors_parsed() if author["email"]])
+                addrs.extend([formataddr((author["name"], author["email"])) for author in submission.authors_parsed() if author["email"]])
                 if submission.submitter_parsed()["email"]: 
                     addrs.append(submission.submitter)
         return addrs

ietf/message/tests.py

@@ -45,7 +45,7 @@ def test_send_plain_announcement(self):
             frm="testmonkey@example.com",
             cc="cc.a@example.com, cc.b@example.com",
             bcc="bcc@example.com",
-            body="Hello World!",
+            body=u"Hello World!",
             content_type="",
             )
 
@@ -73,7 +73,7 @@ def test_send_mime_announcement(self):
             frm="testmonkey@example.com",
             cc="cc.a@example.com, cc.b@example.com",
             bcc="bcc@example.com",
-            body='--NextPart\r\n\r\nA New Internet-Draft is available from the on-line Internet-Drafts directories.\r\n--NextPart\r\nContent-Type: Message/External-body;\r\n\tname="draft-huang-behave-bih-01.txt";\r\n\tsite="ftp.ietf.org";\r\n\taccess-type="anon-ftp";\r\n\tdirectory="internet-drafts"\r\n\r\nContent-Type: text/plain\r\nContent-ID:     <2010-07-30001541.I-D@ietf.org>\r\n\r\n--NextPart--',
+            body=u'--NextPart\r\n\r\nA New Internet-Draft is available from the on-line Internet-Drafts directories.\r\n--NextPart\r\nContent-Type: Message/External-body;\r\n\tname="draft-huang-behave-bih-01.txt";\r\n\tsite="ftp.ietf.org";\r\n\taccess-type="anon-ftp";\r\n\tdirectory="internet-drafts"\r\n\r\nContent-Type: text/plain\r\nContent-ID:     <2010-07-30001541.I-D@ietf.org>\r\n\r\n--NextPart--',
             content_type='Multipart/Mixed; Boundary="NextPart"',
             )
 

ietf/person/models.py

@@ -243,7 +243,18 @@ def formatted_ascii_email(self):
         else:
             return self.address
 
+    def name_and_email(self):
+        "Returns name and email, e.g.: u'Ano Nymous <ano@nymous.org>' "
+        if self.person:
+            return u"%s <%s>" % (self.person.plain_name(), self.address)
+        else:
+            return u"<%s>" % self.address
+
     def formatted_email(self):
+        """
+        Similar to name_and_email(), but with email header-field
+        encoded words (RFC 2047) and quotes as needed.
+        """
         if self.person:
             return formataddr((self.person.plain_name(), self.address))
         else:

ietf/person/tests.py

@@ -11,7 +11,7 @@
 from ietf.person.factories import EmailFactory,PersonFactory
 from ietf.person.models import Person
 from ietf.utils.test_data import make_test_data
-from ietf.utils.test_utils import TestCase, unicontent
+from ietf.utils.test_utils import TestCase
 from ietf.utils.mail import outbox, empty_outbox
 
 
@@ -41,8 +41,7 @@ def test_profile(self):
 
         url = urlreverse("ietf.person.views.profile", kwargs={ "email_or_name": person.plain_name()})
         r = self.client.get(url)
-        self.assertEqual(r.status_code, 200)
-        self.assertIn(person.photo_name(), unicontent(r))
+        self.assertContains(r, person.photo_name(), status_code=200)
         q = PyQuery(r.content)
         self.assertIn("Photo of %s"%person, q("div.bio-text img.bio-photo").attr("alt"))
 

ietf/utils/mail.py

@@ -183,11 +183,8 @@ def send_mail(request, to, frm, subject, template, context, *args, **kwargs):
     return send_mail_text(request, to, frm, subject, txt, *args, **kwargs)
 
 def encode_message(txt):
-    if isinstance(txt, unicode):
-        msg = MIMEText(txt.encode('utf-8'), 'plain', 'UTF-8')
-    else:
-        msg = MIMEText(txt)
-    return msg
+    assert isinstance(txt, unicode)
+    return MIMEText(txt.encode('utf-8'), 'plain', 'UTF-8')
 
 def send_mail_text(request, to, frm, subject, txt, cc=None, extra=None, toUser=False, bcc=None):
     """Send plain text message."""
@@ -307,6 +304,7 @@ def send_mail_mime(request, to, frm, subject, msg, cc=None, extra=None, toUser=F
 def parse_preformatted(preformatted, extra={}, override={}):
     """Parse preformatted string containing mail with From:, To:, ...,"""
     msg = message_from_string(preformatted.encode("utf-8"))
+    msg.set_charset('UTF-8')
 
     for k, v in override.iteritems():
          if k in msg:
@@ -332,7 +330,8 @@ def send_mail_preformatted(request, preformatted, extra={}, override={}):
     extra headers as needed)."""
 
     (msg,headers,bcc) = parse_preformatted(preformatted, extra, override)
-    send_mail_text(request, msg['To'], msg["From"], msg["Subject"], msg.get_payload(), extra=headers, bcc=bcc)
+    txt = msg.get_payload().decode(str(msg.get_charset()))
+    send_mail_text(request, msg['To'], msg["From"], msg["Subject"], txt, extra=headers, bcc=bcc)
     return msg
 
 def send_mail_message(request, message, extra={}):

ietf/utils/tests.py

@@ -52,7 +52,7 @@ class TestSMTPServer(TestCase):
     def test_address_rejected(self):
 
         def send_simple_mail(to):
-            send_mail_text(None, to=to, frm=None, subject="Test for rejection", txt="dummy body")
+            send_mail_text(None, to=to, frm=None, subject="Test for rejection", txt=u"dummy body")
 
         len_before = len(outbox)
         send_simple_mail('good@example.com,poison@example.com')