Add some new encoding tests to mailgw:

- confirm that non-utf8 encodings work
- confirm that non-ASCII headers work (and they didn't but do now)

(roundup.rfc2822 is almost entirely gone from Roundup use now)

Author: Richard Jones

roundup/mailgw.py

@@ -81,6 +81,8 @@ class node. Any parts of other types are each stored in separate files
 import time, random, sys, logging
 import traceback, MimeWriter, rfc822
 
+from email.Header import decode_header
+
 from roundup import configuration, hyperdb, date, password, rfc2822, exceptions
 from roundup.mailer import Mailer, MessageSendError
 from roundup.i18n import _
@@ -261,9 +263,30 @@ def getparts(self):
 
     def getheader(self, name, default=None):
         hdr = mimetools.Message.getheader(self, name, default)
+        if not hdr:
+            return ''
         if hdr:
             hdr = hdr.replace('\n','') # Inserted by rfc822.readheaders
-        return rfc2822.decode_header(hdr)
+        # historically this method has returned utf-8 encoded string
+        l = []
+        for part, encoding in decode_header(hdr):
+            if encoding:
+                part = part.decode(encoding)
+            l.append(part)
+        return ''.join([s.encode('utf-8') for s in l])
+
+    def getaddrlist(self, name):
+        # overload to decode the name part of the address
+        l = []
+        for (name, addr) in mimetools.Message.getaddrlist(self, name):
+            p = []
+            for part, encoding in decode_header(name):
+                if encoding:
+                    part = part.decode(encoding)
+                p.append(part)
+            name = ''.join([s.encode('utf-8') for s in p])
+            l.append((name, addr))
+        return l
 
     def getname(self):
         """Find an appropriate name for this message."""

test/test_mailgw.py

@@ -1,3 +1,4 @@
+# -*- encoding: utf-8 -*-
 #
 # Copyright (c) 2001 Richard Jones, [email protected].
 # This module is free software, and you may redistribute it and/or modify
@@ -23,6 +24,7 @@
 from roundup.mailgw import MailGW, Unauthorized, uidFromAddress, \
     parseContent, IgnoreLoop, IgnoreBulk, MailUsageError, MailUsageHelp
 from roundup import init, instance, password, rfc2822, __version__
+from roundup.anypy.sets_ import set
 
 import db_test_base
 
@@ -1062,6 +1064,29 @@ def testNewUserAuthor(self):
         m.sort()
         self.assertNotEqual(l, m)
 
+    def testNewUserAuthorHighBit(self):
+        l = set(self.db.user.list())
+        # From: name has Euro symbol in it
+        message = '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: =?utf8?b?SOKCrGxsbw==?= <[email protected]>
+To: [email protected]
+Message-Id: <dummy_test_message_id>
+Subject: [issue] Testing...
+
+This is a test submission of a new issue.
+'''
+        p = [
+            self.db.security.getPermission('Create', 'user'),
+            self.db.security.getPermission('Email Access', None),
+        ]
+        self.db.security.role['anonymous'].permissions=p
+        self._handle_mail(message)
+        m = set(self.db.user.list())
+        new = list(m - l)[0]
+        name = self.db.user.get(new, 'realname')
+        self.assertEquals(name, 'H€llo')
+
     def testEnc01(self):
         self.doNewIssue()
         self._handle_mail('''Content-Type: text/plain;
@@ -1102,6 +1127,53 @@ def testEnc01(self):
 ----------
 status: unread -> chatting
 
+_______________________________________________________________________
+Roundup issue tracker <[email protected]>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
+''')
+
+    def testEncNonUTF8(self):
+        self.doNewIssue()
+        self.instance.config.EMAIL_CHARSET = 'iso-8859-1'
+        self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: mary <[email protected]>
+To: [email protected]
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+Subject: [issue1] Testing...
+Content-Type: text/plain;
+        charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+A message with encoding (encoded oe =F6)
+
+''')
+        self.compareMessages(self._get_mail(),
+'''FROM: [email protected]
+TO: [email protected], [email protected]
+Content-Type: text/plain; charset="iso-8859-1"
+Subject: [issue1] Testing...
+To: [email protected], [email protected]
+From: "Contrary, Mary" <[email protected]>
+Reply-To: Roundup issue tracker <[email protected]>
+MIME-Version: 1.0
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
+Content-Transfer-Encoding: quoted-printable
+
+
+Contrary, Mary <[email protected]> added the comment:
+
+A message with encoding (encoded oe =F6)
+
+----------
+status: unread -> chatting
+
 _______________________________________________________________________
 Roundup issue tracker <[email protected]>
 <http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>