Fix subject parsing in mail gateway.

The previous parsing routine would not ensure that arguments are at the
end of the subject and when subject_suffix_parsing was configured to be
'loose' it would truncate the subject when encountering a double prefix,
e.g.  Subject: [frobulated] [frobulatedagain] this part would be lost

Author: schlatterbeck

CHANGES.txt

@@ -293,6 +293,12 @@ Fixed:
   of the tracker html directory. This has been fixed by normalizing
   the path and comparing to the normalized path for the html
   directory. See ``doc/upgrading.txt``. (John Rouillard)
+- Fix subject parsing in mail gateway. The previous parsing routine
+  would not ensure that arguments are at the end of the subject and when
+  subject_suffix_parsing was configured to be 'loose' it would truncate
+  the subject when encountering a double prefix, e.g.
+  Subject: [frobulated] [frobulatedagain] this part would be lost
+  (Ralf Schlatterbeck)
 
 2016-01-11: 1.5.1
 

roundup/mailgw.py

@@ -652,33 +652,33 @@ def parse_subject(self):
 
             tmpsubject = tmpsubject[m.end():]
 
-        # Match the title of the subject
-        # if we've not found a valid classname prefix then force the
-        # scanning to handle there being a leading delimiter
-        title_re = r'(?P<title>%s[^%s]*)'%(
-            not self.matches['classname'] and '.' or '', delim_open)
-        m = re.match(title_re, tmpsubject.strip(), re.IGNORECASE)
+        # Match any arguments specified *from the end*
+        # Optionally match and strip quote at the end that dumb mailers
+        # may put there, e.g.
+        #      Re: "[issue1] bla blah [<args>]"
+        q = ''
+        if self.matches['quote']:
+            q = '"?'
+        args_re = r'(?P<argswhole>%s(?P<args>[^%s]*)%s)%s$'%(delim_open,
+            delim_close, delim_close, q)
+        m = re.search(args_re, tmpsubject.strip(), re.IGNORECASE|re.VERBOSE)
         if m:
             self.matches.update(m.groupdict())
-            tmpsubject = tmpsubject[len(self.matches['title']):] # Consume title
-
-        if self.matches['title']:
-            self.matches['title'] = self.matches['title'].strip()
+            tmpsubject = tmpsubject [:m.start()]
         else:
-            self.matches['title'] = ''
+            self.matches['argswhole'] = self.matches['args'] = None
+
+        # The title of the subject is the remaining tmpsubject.
+        self.matches ['title'] = tmpsubject.strip ()
 
         # strip off the quotes that dumb emailers put around the subject, like
         #      Re: "[issue1] bla blah"
-        if self.matches['quote'] and self.matches['title'].endswith('"'):
+        # but only if we didn't match arguments at the end (which would
+        # already have consumed the quote after the subject)
+        if self.matches['quote'] and not self.matches['argswhole'] \
+           and self.matches['title'].endswith('"'):
             self.matches['title'] = self.matches['title'][:-1]
 
-        # Match any arguments specified
-        args_re = r'(?P<argswhole>%s(?P<args>.+?)%s)?'%(delim_open,
-            delim_close)
-        m = re.search(args_re, tmpsubject.strip(), re.IGNORECASE|re.VERBOSE)
-        if m:
-            self.matches.update(m.groupdict())
-
     def rego_confirm(self):
         ''' Check for registration OTK and confirm the registration if found
         '''

test/test_mailgw.py

@@ -3054,6 +3054,24 @@ def testInvalidClassLooseReply(self):
 Reply-To: [email protected]
 Message-Id: <dummy_test_message_id>
 
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.issue.get(nodeid, 'title'),
+            '[frobulated] testing')
+
+    def testInvalidClassLooseReplyQuoted(self):
+        self.instance.config.MAILGW_SUBJECT_PREFIX_PARSING = 'loose'
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <[email protected]>
+To: [email protected]
+Subject: Re: "[frobulated] testing"
+Cc: [email protected]
+Reply-To: [email protected]
+Message-Id: <dummy_test_message_id>
+
+Dumb mailers may put quotes around the subject after the reply prefix,
+e.g. Re: "[issue1] bla bla"
 ''')
         assert not os.path.exists(SENDMAILDEBUG)
         self.assertEqual(self.db.issue.get(nodeid, 'title'),
@@ -3091,6 +3109,23 @@ def testClassLooseOK(self):
         assert not os.path.exists(SENDMAILDEBUG)
         self.assertEqual(self.db.keyword.get('1', 'name'), 'Bar')
 
+    def testDoublePrefixLoose(self):
+        self.instance.config.MAILGW_SUBJECT_PREFIX_PARSING = 'loose'
+        self.instance.config.MAILGW_SUBJECT_SUFFIX_PARSING = 'loose'
+        nodeid = self._handle_mail('''Content-Type: text/plain;
+  charset="iso-8859-1"
+From: Chef <[email protected]>
+To: [email protected]
+Subject: [frobulated] [frobulatedagain] testing stuff after double prefix
+Cc: [email protected]
+Reply-To: [email protected]
+Message-Id: <dummy_test_message_id>
+
+''')
+        assert not os.path.exists(SENDMAILDEBUG)
+        self.assertEqual(self.db.issue.get(nodeid, 'title'),
+            '[frobulated] [frobulatedagain] testing stuff after double prefix')
+
     def testClassStrictInvalid(self):
         self.instance.config.MAILGW_SUBJECT_PREFIX_PARSING = 'strict'
         self.instance.config.MAILGW_DEFAULT_CLASS = ''