Fix issue934009: Have New Issues Submitted By Email *Not* Change Body!

The mailgw config options: keep_quoted_text and leave_body_unchanged
can now have a new values: new. If set to new, keep_quoted_text acts
like yes if the message is starting a new issue. Otherise it strips
quoted text.

This allows somebody to start a new issue by forwarding
a threaded email (with multiple quoted parts) into roundup and
keeping all the quoted parts.  If leave_body_unchanged is set to
new, even the signature on the email that starts a new issue will be
preserved.

Author: rouilj

CHANGES.txt

@@ -111,7 +111,16 @@ Features:
   describe the purpose of the directory. It also says the detectors
   are provided on an as-is basis and may not work. Detector by W.
   Trevor King (wking), rest by John Rouillard.
-
+- issue934009: Have New Issues Submitted By Email *Not* Change Body!
+  The mailgw config options: keep_quoted_text and leave_body_unchanged
+  can now have a new values: new. If set to new, keep_quoted_text acts
+  like yes if the message is starting a new issue. Otherise it strips
+  quoted text. This allows somebody to start a new issue by forwarding
+  a threaded email (with multiple quoted parts) into roundup and
+  keeping all the quoted parts.  If leave_body_unchanged is set to
+  new, even the signature on the email that starts a new issue will be
+  preserved.
+  
 Fixed:
 
 - issue1615201: Optionally restore the original (version 0.6) mailgw 

roundup/configuration.py

@@ -293,6 +293,19 @@ def str2value(self, value):
         else:
             raise OptionValueError(self, value, self.class_description)
 
+class EmailBodyOption(Option):
+
+    """When to replace message body or strip quoting: always, never or for new items only"""
+
+    class_description = "Allowed values: yes, no, new"
+
+    def str2value(self, value):
+        _val = value.lower()
+        if _val in ("yes", "no", "new"):
+            return _val
+        else:
+            raise OptionValueError(self, value, self.class_description)
+
 class IsolationOption(Option):
     """Database isolation levels"""
 
@@ -760,15 +773,18 @@ def str2value(self, value):
             "actor from being exposed at mail archives, etc."),
     ), "Outgoing email options.\nUsed for nozy messages and approval requests"),
     ("mailgw", (
-        (BooleanOption, "keep_quoted_text", "yes",
+        (EmailBodyOption, "keep_quoted_text", "yes",
             "Keep email citations when accepting messages.\n"
-            "Setting this to \"no\" strips out \"quoted\" text"
-            " from the message.\n"
+            "Setting this to \"no\" strips out \"quoted\" text\n"
+            "from the message. Setting this to \"new\" keeps quoted\n"
+            "text only if a new issue is being created.\n"
             "Signatures are also stripped.",
             ["EMAIL_KEEP_QUOTED_TEXT"]),
-        (BooleanOption, "leave_body_unchanged", "no",
-            "Preserve the email body as is - that is,\n"
-            "keep the citations _and_ signatures.",
+        (EmailBodyOption, "leave_body_unchanged", "no",
+            "Setting this to \"yes\" preserves the email body\n"
+            "as is - that is, keep the citations _and_ signatures.\n"
+            "Setting this to \"new\" keeps the body only if we are\n"
+            "creating a new issue.",
             ["EMAIL_LEAVE_BODY_UNCHANGED"]),
         (Option, "default_class", "issue",
             "Default class to use in the mailgw\n"

roundup/mailgw.py

@@ -1138,9 +1138,8 @@ def create_msg(self):
 Roundup requires the submission to be plain text. The message parser could
 not find a text/plain part to use.
 """)
-
         # parse the body of the message, stripping out bits as appropriate
-        summary, content = parseContent(self.content, config=self.config)
+        summary, content = parseContent(self.content, config=self.config, is_new_issue = not bool(self.nodeid))
         content = content.strip()
 
         if content:
@@ -1782,7 +1781,7 @@ def uidFromAddress(db, address, create=1, **user_props):
     else:
         return 0
 
-def parseContent(content, keep_citations=None, keep_body=None, config=None):
+def parseContent(content, keep_citations=None, keep_body=None, config=None, is_new_issue=False):
     """Parse mail message; return message summary and stripped content
 
     The message body is divided into sections by blank lines.
@@ -1806,8 +1805,30 @@ def parseContent(content, keep_citations=None, keep_body=None, config=None):
         config = configuration.CoreConfig()
     if keep_citations is None:
         keep_citations = config["MAILGW_KEEP_QUOTED_TEXT"]
+        if keep_citations == "new":
+            # don't strip citations if we are a new issue
+            if is_new_issue:
+                keep_citations = True
+            else:
+                keep_citations = False
+        elif keep_citations == "yes":
+            keep_citations = True
+        else:
+            keep_citations = False
+
     if keep_body is None:
         keep_body = config["MAILGW_LEAVE_BODY_UNCHANGED"]
+        if keep_body == "new":
+            # don't strip citations if we are a new issue
+            if is_new_issue:
+                keep_body = True
+            else:
+                keep_body = False
+        elif keep_body == "yes":
+            keep_body = True
+        else:
+            keep_body = False
+
     eol = config["MAILGW_EOL_RE"]
     signature = config["MAILGW_SIGN_RE"]
     original_msg = config["MAILGW_ORIGMSG_RE"]

test/test_mailgw.py

@@ -2547,6 +2547,75 @@ def testEmailQuoting(self):
         self.innerTestQuoting(self.firstquotingtest, '''This is a followup
 ''', 'This is a followup')
 
+    def testEmailQuotingNewIsNew(self):
+        self.instance.config.EMAIL_KEEP_QUOTED_TEXT = 'new'
+        # create the message, remove the prefix from subject
+        testmessage=self.firstquotingtest.replace(" Re: [issue1]", "")
+        nodeid = self._handle_mail(testmessage)
+
+        msgs = self.db.issue.get(nodeid, 'messages')
+        # validate content and summary
+        content = self.db.msg.get(msgs[0], 'content')
+        self.assertEqual(content, '''Blah blah wrote:
+> Blah bklaskdfj sdf asdf jlaskdf skj sdkfjl asdf
+>  skdjlkjsdfalsdkfjasdlfkj dlfksdfalksd fj
+>
+
+This is a followup''')
+
+        summary = self.db.msg.get(msgs[0], 'summary')
+        self.assertEqual(summary,  '''This is a followup''')
+
+    def testEmailQuotingNewIsFollowup(self):
+        self.instance.config.EMAIL_KEEP_QUOTED_TEXT = 'new'
+        # create issue1 that we can followup on
+        self.doNewIssue()
+        # add the second message to the issue
+        nodeid = self._handle_mail(self.firstquotingtest)
+        msgs = self.db.issue.get(nodeid, 'messages')
+        # check second message for accuracy
+        content = self.db.msg.get(msgs[1], 'content')
+        summary = self.db.msg.get(msgs[1], 'summary')
+        self.assertEqual(content,  '''This is a followup''')
+        self.assertEqual(summary,  '''This is a followup''')
+
+    def testEmailReplaceBodyNewIsNew(self):
+        mysig = "--\nmy sig\n\n"
+        self.instance.config.EMAIL_LEAVE_BODY_UNCHANGED = 'new'
+        # create the message, remove the prefix from subject
+        testmessage=self.firstquotingtest.replace(" Re: [issue1]", "") + mysig
+        nodeid = self._handle_mail(testmessage)
+
+        msgs = self.db.issue.get(nodeid, 'messages')
+        # validate content and summary
+        content = self.db.msg.get(msgs[0], 'content')
+        self.assertEqual(content, '''Blah blah wrote:
+> Blah bklaskdfj sdf asdf jlaskdf skj sdkfjl asdf
+>  skdjlkjsdfalsdkfjasdlfkj dlfksdfalksd fj
+>
+
+This is a followup\n''' + mysig[:-2])
+        # the :-2 requrement to strip the trailing newlines is probably a bug
+        # somewhere mailgw has right content maybe trailing \n are stripped by
+        # msg or something.
+
+        summary = self.db.msg.get(msgs[0], 'summary')
+        self.assertEqual(summary,  '''This is a followup''')
+
+    def testEmailReplaceBodyNewIsFollowup(self):
+        mysig = "\n--\nmy sig\n"
+        self.instance.config.EMAIL_LEAVE_BODY_UNCHANGED = 'new'
+        # create issue1 that we can followup on
+        self.doNewIssue()
+        # add the second message to the issue
+        nodeid = self._handle_mail(self.firstquotingtest + mysig)
+        msgs = self.db.issue.get(nodeid, 'messages')
+        # check second message for accuracy
+        content = self.db.msg.get(msgs[1], 'content')
+        summary = self.db.msg.get(msgs[1], 'summary')
+        self.assertEqual(content,  '''Blah blah wrote:\n> Blah bklaskdfj sdf asdf jlaskdf skj sdkfjl asdf\n>  skdjlkjsdfalsdkfjasdlfkj dlfksdfalksd fj\n>\n\nThis is a followup''')
+        self.assertEqual(summary,  '''This is a followup''')
+
     def testEmailQuotingRemove(self):
         self.instance.config.EMAIL_KEEP_QUOTED_TEXT = 'yes'
         self.innerTestQuoting(self.firstquotingtest, '''Blah blah wrote: