issue2550826 IOError in detector causes apache 'premature end of script headers' error

Capture all exceptions from auditors/reactors and raise a
DetectorError instead. This allows failures like IOErrors from the
detectors (e.g. unable to access files) to be handled.  Previously an
IOError just resulted in no output (premature end of headers under
apache). Problem diagnosed and initial patch created by Tom Ekberg
(tekberg).

Patch application/mods and testing by rouilj.

Author: rouilj

CHANGES.txt

@@ -69,6 +69,13 @@ Fixed:
 - issue2550907 Fix errors when creating documentation. Work done by
   Peter Funk (pefu). (Applied by John Rouillard with small change
   omitting obsolete security.txt.)
+- issue2550826 Capture all exceptions from auditors/reactors and
+  raise a DetectorError instead. This allows failures like IOErrors
+  from the detectors (e.g. unable to access files) to be handled.
+  Previously an IOError just resulted in no output (premature end of
+  headers under apache). Problem diagnosed and initial patch created by
+  Tom Ekberg (tekberg). Further testing and patch change done by
+  John Rouillard.
 
 2016-01-11: 1.5.1
 

roundup/cgi/client.py

@@ -19,7 +19,7 @@
 from roundup.exceptions import LoginError, Reject, RejectRaw, Unauthorised
 from roundup.cgi.exceptions import (
     FormError, NotFound, NotModified, Redirect, SendFile, SendStaticFile,
-    SeriousError)
+    DetectorError, SeriousError)
 from roundup.cgi.form_parser import FormParser
 from roundup.mailer import Mailer, MessageSendError, encode_quopri
 from roundup.cgi import accept_language
@@ -572,6 +572,15 @@ def inner_main(self):
             # OpenSSL.SSL.SysCallError is similar to IOError above
             # may happen during write_html and serve_file, too.
             pass
+        except DetectorError as e:
+            if not self.instance.config.WEB_DEBUG:
+                # run when we are not in debug mode, so errors
+                # go to admin too.
+                self.send_error_to_admin(e.subject, e.html, e.txt)
+                self.write_html(e.html)
+            else:
+                # in debug mode, only write error to screen.
+                self.write_html(e.html)
         except:
             # Something has gone badly wrong.  Therefore, we should
             # make sure that the response code indicates failure.

roundup/cgi/exceptions.py

@@ -18,6 +18,14 @@ class NotFound(HTTPException):
 class NotModified(HTTPException):
     pass
 
+class DetectorError(Exception):
+    """Raised when a detector throws an exception.
+Contains details of the exception."""
+    def __init__(self, subject, html, txt):
+        self.subject = subject
+        self.html = html
+        self.txt = txt
+
 class FormError(ValueError):
     """An 'expected' exception occurred during form parsing.
 

roundup/hyperdb.py

@@ -21,12 +21,14 @@
 __docformat__ = 'restructuredtext'
 
 # standard python modules
-import os, re, shutil, weakref
+import os, re, shutil, sys, weakref
+import traceback
 
 # roundup modules
 import date, password
 from support import ensureParentsExist, PrioList
 from roundup.i18n import _
+from roundup.cgi.exceptions import DetectorError
 
 #
 # Types
@@ -1290,7 +1292,16 @@ def audit(self, event, detector, priority = 100):
     def fireAuditors(self, event, nodeid, newvalues):
         """Fire all registered auditors"""
         for prio, name, audit in self.auditors[event]:
-            audit(self.db, self, nodeid, newvalues)
+            try:
+                audit(self.db, self, nodeid, newvalues)
+            except Exception as e:
+                tb = traceback.format_exc()
+                html = ("<h1>Traceback</h1>" + str(tb).replace('\n', '<br>').
+                        replace(' ', '&nbsp;'))
+                txt = 'Caught exception %s: %s\n%s' % (str(type(e)), e, tb)
+                exc_info = sys.exc_info()
+                subject = "Error: %s" % exc_info[1]
+                raise DetectorError(subject, html, txt)
 
     def react(self, event, detector, priority = 100):
         """Register a reactor detector"""
@@ -1299,7 +1310,16 @@ def react(self, event, detector, priority = 100):
     def fireReactors(self, event, nodeid, oldvalues):
         """Fire all registered reactors"""
         for prio, name, react in self.reactors[event]:
-            react(self.db, self, nodeid, oldvalues)
+            try:
+                react(self.db, self, nodeid, oldvalues)
+            except Exception as e:
+                tb = traceback.format_exc()
+                html = ("<h1>Traceback</h1>" + str(tb).replace('\n', '<br>').
+                        replace(' ', '&nbsp;'))
+                txt = 'Caught exception %s: %s\n%s' % (str(type(e)), e, tb)
+                exc_info = sys.exc_info()
+                subject = "Error: %s" % exc_info[1]
+                raise DetectorError(subject, html, txt)
 
     #
     # import / export support