Yet another fix to the mail gateway...

...messages got *all* files of an issue, not just the new ones. Thanks
to Rafal Bisingier for reporting and proposing a fix. The regression
test was updated.  Fix version numbers in upgrade documentation, the
file-unlink defect was in 1.4.17 not 1.4.16. Thanks to Rafal
Bisingier.

Author: Ralf Schlatterbeck

CHANGES.txt

@@ -13,6 +13,11 @@ Fixed:
   (Ralf)
 - Fix version numbers in upgrade documentation, the file-unlink defect
   was in 1.4.17 not 1.4.16. Thanks to Rafal Bisingier. (Ralf)
+- Fix encoded email header parsing if multiple encoded and non-encoded
+  parts are present. RFC2047 specifies that spacing is removed only
+  between encoded parts, we always removed the space. Note that this bug
+  was present before mail gateway refactoring :-) Thanks for thorough
+  testing of mail gateway code by Rafal Bisingier. (Ralf)
 
 2011-05-29 1.4.18 (r4610)
 

roundup/mailgw.py

@@ -247,6 +247,22 @@ def getparts(self):
             parts.append(part)
         return parts
 
+    def _decode_header_to_utf8(self, hdr):
+        l = []
+        prev_encoded = False
+        for part, encoding in decode_header(hdr):
+            if encoding:
+                part = part.decode(encoding)
+            # RFC 2047 specifies that between encoded parts spaces are
+            # swallowed while at the borders from encoded to non-encoded
+            # or vice-versa we must preserve a space. Multiple adjacent
+            # non-encoded parts should not occur.
+            if l and prev_encoded != bool(encoding):
+                l.append(' ')
+            prev_encoded = bool(encoding)
+            l.append(part)
+        return ''.join([s.encode('utf-8') for s in l])
+
     def getheader(self, name, default=None):
         hdr = mimetools.Message.getheader(self, name, default)
         # TODO are there any other False values possible?
@@ -257,24 +273,13 @@ def getheader(self, name, default=None):
             return ''
         if hdr:
             hdr = hdr.replace('\n','') # Inserted by rfc822.readheaders
-        # historically this method has returned utf-8 encoded string
-        l = []
-        for part, encoding in decode_header(hdr):
-            if encoding:
-                part = part.decode(encoding)
-            l.append(part)
-        return ''.join([s.encode('utf-8') for s in l])
+        return self._decode_header_to_utf8(hdr)
 
     def getaddrlist(self, name):
         # overload to decode the name part of the address
         l = []
         for (name, addr) in mimetools.Message.getaddrlist(self, name):
-            p = []
-            for part, encoding in decode_header(name):
-                if encoding:
-                    part = part.decode(encoding)
-                p.append(part)
-            name = ''.join([s.encode('utf-8') for s in p])
+            name = self._decode_header_to_utf8(name)
             l.append((name, addr))
         return l
 

test/test_mailgw.py

@@ -1823,6 +1823,35 @@ def testNewUserAuthorEncodedName(self):
         name = self.db.user.get(new, 'realname')
         self.assertEquals(name, 'H€llo')
 
+    def testNewUserAuthorMixedEncodedName(self):
+        l = set(self.db.user.list())
+        # From: name has Euro symbol in it
+        message = '''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Firstname =?utf-8?b?w6TDtsOf?= Last <[email protected]>
+To: [email protected]
+Message-Id: <dummy_test_message_id>
+Subject: [issue] Test =?utf-8?b?w4TDlsOc?= umlauts
+ X1
+ X2
+
+This is a test submission of a new issue.
+'''
+        p = [
+            self.db.security.getPermission('Register', 'user'),
+            self.db.security.getPermission('Email Access', None),
+            self.db.security.getPermission('Create', 'issue'),
+            self.db.security.getPermission('Create', 'msg'),
+        ]
+        self.db.security.role['anonymous'].permissions = p
+        self._handle_mail(message)
+        title = self.db.issue.get('1', 'title')
+        self.assertEquals(title, 'Test \xc3\x84\xc3\x96\xc3\x9c umlauts X1 X2')
+        m = set(self.db.user.list())
+        new = list(m - l)[0]
+        name = self.db.user.get(new, 'realname')
+        self.assertEquals(name, 'Firstname \xc3\xa4\xc3\xb6\xc3\x9f Last')
+
     def testUnknownUser(self):
         l = set(self.db.user.list())
         message = '''Content-Type: text/plain;