Mail gateway fixes and improvements.

- new mailgw config item unpack_rfc822 that unpacks message attachments
  of type message/rfc822 and attaches the individual parts instead of
  attaching the whole message/rfc822 attachment to the roundup issue.
- Fix handling of incoming message/rfc822 attachments. These resulted in
  a weird mail usage error because the email module threw a TypeError
  which roundup interprets as a Reject exception. Fixes issue2550667.
  Added regression tests for message/rfc822 attachments with and without
  configured unpacking (mailgw unpack_rfc822, see Features above)
  Thanks to Benni Bärmann for reporting.

Author: Ralf Schlatterbeck

CHANGES.txt

@@ -17,6 +17,9 @@ Features:
   timeout of 30 seconds configurable. This is the time a client waits
   for the locked database to become free before giving up. Used only for
   SQLite backend.
+- new mailgw config item unpack_rfc822 that unpacks message attachments
+  of type message/rfc822 and attaches the individual parts instead of
+  attaching the whole message/rfc822 attachment to the roundup issue.
 
 Fixed:
 
@@ -33,6 +36,12 @@ Fixed:
 - Fix charset of first text-part of outgoing multipart messages, thanks Dirk
   Geschke for reporting, see
   http://thread.gmane.org/gmane.comp.bug-tracking.roundup.user/10223
+- Fix handling of incoming message/rfc822 attachments. These resulted in
+  a weird mail usage error because the email module threw a TypeError
+  which roundup interprets as a Reject exception. Fixes issue2550667.
+  Added regression tests for message/rfc822 attachments with and without
+  configured unpacking (mailgw unpack_rfc822, see Features above)
+  Thanks to Benni B�rmann for reporting.
 
 
 2010-07-12 1.4.15

roundup/configuration.py

@@ -748,6 +748,10 @@ def str2value(self, value):
             "Regular expression matching end of line."),
         (RegExpOption, "blankline_re", r"[\r\n]+\s*[\r\n]+",
             "Regular expression matching a blank line."),
+        (BooleanOption, "unpack_rfc822", "no",
+            "Unpack attached messages (encoded as message/rfc822 in MIME)\n"
+            "as multiple parts attached as files to the issue, if not\n"
+            "set we handle message/rfc822 attachments as a single file."),
         (BooleanOption, "ignore_alternatives", "no",
             "When parsing incoming mails, roundup uses the first\n"
             "text/plain part it finds. If this part is inside a\n"

roundup/mailgw.py

@@ -27,6 +27,9 @@ class node. Any parts of other types are each stored in separate files
    and given "file" class nodes that are linked to the "msg" node.
  . In a multipart/alternative message or part, we look for a text/plain
    subpart and ignore the other parts.
+ . A message/rfc822 is treated similar tomultipart/mixed (except for
+   special handling of the first text part) if unpack_rfc822 is set in
+   the mailgw config section.
 
 Summary
 -------
@@ -277,12 +280,17 @@ def getaddrlist(self, name):
 
     def getname(self):
         """Find an appropriate name for this message."""
+        name = None
         if self.gettype() == 'message/rfc822':
             # handle message/rfc822 specially - the name should be
             # the subject of the actual e-mail embedded here
+            # we add a '.eml' extension like other email software does it
             self.fp.seek(0)
-            name = Message(self.fp).getheader('subject')
-        else:
+            s = cStringIO.StringIO(self.getbody())
+            name = Message(s).getheader('subject')
+            if name:
+                name = name + '.eml'
+        if not name:
             # try name on Content-Type
             name = self.getparam('name')
             if not name:
@@ -355,8 +363,11 @@ def getbody(self):
     #   flagging.
     # multipart/form-data:
     #   For web forms only.
+    # message/rfc822:
+    #   Only if configured in [mailgw] unpack_rfc822
 
-    def extract_content(self, parent_type=None, ignore_alternatives = False):
+    def extract_content(self, parent_type=None, ignore_alternatives=False,
+        unpack_rfc822=False):
         """Extract the body and the attachments recursively.
 
            If the content is hidden inside a multipart/alternative part,
@@ -374,7 +385,7 @@ def extract_content(self, parent_type=None, ignore_alternatives = False):
             ig = ignore_alternatives and not content_found
             for part in self.getparts():
                 new_content, new_attach = part.extract_content(content_type,
-                    not content and ig)
+                    not content and ig, unpack_rfc822)
 
                 # If we haven't found a text/plain part yet, take this one,
                 # otherwise make it an attachment.
@@ -399,6 +410,13 @@ def extract_content(self, parent_type=None, ignore_alternatives = False):
                 attachments.extend(new_attach)
             if ig and content_type == 'multipart/alternative' and content:
                 attachments = []
+        elif unpack_rfc822 and content_type == 'message/rfc822':
+            s = cStringIO.StringIO(self.getbody())
+            m = Message(s)
+            ig = ignore_alternatives and not content
+            new_content, attachments = m.extract_content(m.gettype(), ig,
+                unpack_rfc822)
+            attachments.insert(0, m.text_as_attachment())
         elif (parent_type == 'multipart/signed' and
               content_type == 'application/pgp-signature'):
             # ignore it so it won't be saved as an attachment
@@ -1276,7 +1294,8 @@ def pgp_role():
 encrypted.""")
         # now handle the body - find the message
         ig = self.instance.config.MAILGW_IGNORE_ALTERNATIVES
-        content, attachments = message.extract_content(ignore_alternatives = ig)
+        content, attachments = message.extract_content(ignore_alternatives=ig,
+            unpack_rfc822 = self.instance.config.MAILGW_UNPACK_RFC822)
         if content is None:
             raise MailUsageError, _("""
 Roundup requires the submission to be plain text. The message parser could

roundup/roundupdb.py

@@ -30,6 +30,7 @@
 from email.Header import Header
 from email.MIMEText import MIMEText
 from email.MIMEBase import MIMEBase
+from email.parser   import FeedParser
 
 from roundup import password, date, hyperdb
 from roundup.i18n import _
@@ -492,6 +493,12 @@ def send_message(self, issueid, msgid, note, sendto, from_address=None,
                         else:
                             part = MIMEText(content)
                             part['Content-Transfer-Encoding'] = '7bit'
+                    elif mime_type == 'message/rfc822':
+                        main, sub = mime_type.split('/')
+                        p = FeedParser()
+                        p.feed(content)
+                        part = MIMEBase(main, sub)
+                        part.set_payload([p.close()])
                     else:
                         # some other type, so encode it
                         if not mime_type:
@@ -503,7 +510,8 @@ def send_message(self, issueid, msgid, note, sendto, from_address=None,
                         part = MIMEBase(main, sub)
                         part.set_payload(content)
                         Encoders.encode_base64(part)
-                    part['Content-Disposition'] = 'attachment;\n filename="%s"'%name
+                    cd = 'Content-Disposition'
+                    part[cd] = 'attachment;\n filename="%s"'%name
                     message.attach(part)
 
             else:

test/test_mailgw.py

@@ -480,6 +480,54 @@ def testNewIssueNoAuthorEmail(self):
 
 <html>umlaut =E4=F6=FC=C4=D6=DC=DF</html>
 
+--001485f339f8f361fb049188dbba--
+'''
+
+    multipart_msg_rfc822 = '''From: mary <[email protected]>
+To: [email protected]
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+Subject: [issue1] Testing...
+Content-Type: multipart/mixed; boundary=001485f339f8f361fb049188dbba
+
+This is a multi-part message in MIME format.
+--001485f339f8f361fb049188dbba
+Content-Type: text/plain; charset=ISO-8859-15
+Content-Transfer-Encoding: 7bit
+
+First part: Text
+
+--001485f339f8f361fb049188dbba
+Content-Type: message/rfc822; name="Fwd: Original email subject.eml"
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="Fwd: Original email subject.eml"
+
+Message-Id: <followup_dummy_id_2>
+In-Reply-To: <dummy_test_message_id_2>
+MIME-Version: 1.0
+Subject: Fwd: Original email subject
+Date: Mon, 23 Aug 2010 08:23:33 +0200
+Content-Type: multipart/alternative; boundary="090500050101020406060002"
+
+This is a multi-part message in MIME format.
+--090500050101020406060002
+Content-Type: text/plain; charset=ISO-8859-15; format=flowed
+Content-Transfer-Encoding: 7bit
+
+some text in inner email
+========================
+
+--090500050101020406060002
+Content-Type: text/html; charset=ISO-8859-15
+Content-Transfer-Encoding: 7bit
+
+<html>
+some text in inner email
+========================
+</html>
+
+--090500050101020406060002--
+
 --001485f339f8f361fb049188dbba--
 '''
 
@@ -746,6 +794,105 @@ def testMultipartCharsetLatin1AttachFile(self):
 --utf-8--
 ''')
 
+    def testMultipartRFC822(self):
+        self.doNewIssue()
+        self._handle_mail(self.multipart_msg_rfc822)
+        messages = self.db.issue.get('1', 'messages')
+        messages.sort()
+        msg = self.db.msg.getnode (messages[-1])
+        assert(len(msg.files) == 1)
+        name = "Fwd: Original email subject.eml"
+        for n, id in enumerate (msg.files):
+            f = self.db.file.getnode (id)
+            self.assertEqual(f.name, name)
+        self.assertEqual(msg.content, 'First part: Text')
+        self.compareMessages(self._get_mail(),
+'''TO: [email protected], [email protected]
+Content-Type: text/plain; charset="utf-8"
+Subject: [issue1] Testing...
+To: [email protected], [email protected]
+From: "Contrary, Mary" <[email protected]>
+Reply-To: Roundup issue tracker
+ <[email protected]>
+MIME-Version: 1.0
+Message-Id: <followup_dummy_id>
+In-Reply-To: <dummy_test_message_id>
+X-Roundup-Name: Roundup issue tracker
+X-Roundup-Loop: hello
+X-Roundup-Issue-Status: chatting
+X-Roundup-Issue-Files: Fwd: Original email subject.eml
+Content-Transfer-Encoding: quoted-printable
+
+
+--utf-8
+MIME-Version: 1.0
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: quoted-printable
+
+
+Contrary, Mary <[email protected]> added the comment:
+
+First part: Text
+
+----------
+status: unread -> chatting
+
+_______________________________________________________________________
+Roundup issue tracker <[email protected]>
+<http://tracker.example/cgi-bin/roundup.cgi/bugs/issue1>
+_______________________________________________________________________
+--utf-8
+Content-Type: message/rfc822
+MIME-Version: 1.0
+Content-Disposition: attachment;
+ filename="Fwd: Original email subject.eml"
+
+Message-Id: <followup_dummy_id_2>
+In-Reply-To: <dummy_test_message_id_2>
+MIME-Version: 1.0
+Subject: Fwd: Original email subject
+Date: Mon, 23 Aug 2010 08:23:33 +0200
+Content-Type: multipart/alternative; boundary="090500050101020406060002"
+
+This is a multi-part message in MIME format.
+--090500050101020406060002
+Content-Type: text/plain; charset=ISO-8859-15; format=flowed
+Content-Transfer-Encoding: 7bit
+
+some text in inner email
+========================
+
+--090500050101020406060002
+Content-Type: text/html; charset=ISO-8859-15
+Content-Transfer-Encoding: 7bit
+
+<html>
+some text in inner email
+========================
+</html>
+
+--090500050101020406060002--
+
+--utf-8--
+''')
+
+    def testMultipartRFC822Unpack(self):
+        self.doNewIssue()
+        self.db.config.MAILGW_UNPACK_RFC822 = True
+        self._handle_mail(self.multipart_msg_rfc822)
+        messages = self.db.issue.get('1', 'messages')
+        messages.sort()
+        msg = self.db.msg.getnode (messages[-1])
+        self.assertEqual(len(msg.files), 2)
+        t = 'some text in inner email\n========================\n'
+        content = {0 : t, 1 : '<html>\n' + t + '</html>\n'}
+        for n, id in enumerate (msg.files):
+            f = self.db.file.getnode (id)
+            self.assertEqual(f.name, 'unnamed')
+            if n in content :
+                self.assertEqual(f.content, content [n])
+        self.assertEqual(msg.content, 'First part: Text')
+
     def testSimpleFollowup(self):
         self.doNewIssue()
         self._handle_mail('''Content-Type: text/plain;