Handle UnicodeDecodeError in file class when file contents are not

text (e.g. jpg). This shows up in python3 due to conversion from bytes
to string. This allows binary file retrieval via the rest interface to
complete.

Author: rouilj

CHANGES.txt

@@ -126,6 +126,8 @@ Fixed:
   C�dric Krier, John Rouillard)
 - issue2550144 - fix use of undefined icing macro in devel
   template. Replace with frame macro. (C�dric Krier)
+- handle UnicodeDecodeError in file class when file contents are
+  not text (e.g. jpg). (John Rouillard)
 
 2018-07-13 1.6.0
 

roundup/backends/back_anydbm.py

@@ -36,6 +36,8 @@
 
 from roundup.backends.indexer_common import get_indexer
 
+from hashlib import md5
+
 def db_exists(config):
     # check for the user db
     for db in 'nodes.user nodes.user.db'.split():
@@ -2195,6 +2197,15 @@ def get(self, nodeid, propname, default=_marker, cache=1):
                 # XXX by catching this we don't see an error in the log.
                 return 'ERROR reading file: %s%s\n%s\n%s'%(
                         self.classname, nodeid, poss_msg, strerror)
+            except UnicodeDecodeError as e:
+                # if content is not text (e.g. jpeg file) we get
+                # unicode error trying to convert to string in python 3.
+                # trap it and supply an error message. Include md5sum
+                # of content as this string is included in the etag
+                # calculation of the object.
+                return ('%s%s is not text, retrieve using '
+                        'binary_content property. mdsum: %s')%(self.classname,
+                   nodeid, md5(self.db.getfile(self.classname, nodeid, None)).hexdigest())
         elif propname == 'binary_content':
             return self.db.getfile(self.classname, nodeid, None)
 

roundup/backends/rdbms_common.py

@@ -71,6 +71,7 @@
 from roundup.backends.back_anydbm import compile_expression
 from roundup.anypy.strings import b2s, bs2b, us2s, repr_export, eval_import
 
+from hashlib import md5
 
 # dummy value meaning "argument not passed"
 _marker = []
@@ -3079,6 +3080,15 @@ def get(self, nodeid, propname, default=_marker, cache=1):
                 # BUG: by catching this we donot see an error in the log.
                 return 'ERROR reading file: %s%s\n%s\n%s'%(
                         self.classname, nodeid, poss_msg, strerror)
+            except UnicodeDecodeError as e:
+                # if content is not text (e.g. jpeg file) we get
+                # unicode error trying to convert to string in python 3.
+                # trap it and supply an error message. Include md5sum
+                # of content as this string is included in the etag
+                # calculation of the object.
+                return ('%s%s is not text, retrieve using '
+                        'binary_content property. mdsum: %s')%(self.classname,
+                   nodeid, md5(self.db.getfile(self.classname, nodeid, None)).hexdigest())
         elif propname == 'binary_content':
             return self.db.getfile(self.classname, nodeid, None)
 

test/db_test_base.py

@@ -304,6 +304,32 @@ def testStringUnicode(self):
         self.assertEqual(self.db.file.get(nid, 'content'), ustr)
         self.assertEqual(self.db.file.get(nid, 'binary_content'), s2b(ustr))
 
+    def testStringBinary(self):
+        ''' Create file with binary content that is not able
+            to be interpreted as unicode. Try to cause file module
+            trigger and handle UnicodeDecodeError
+            and get valid output
+        '''
+        # test set & retrieve
+        bstr = b'\x00\xF0\x34\x33' # random binary data
+
+        # test set & retrieve (this time for file contents)
+        nid = self.db.file.create(content=bstr)
+        print(nid)
+        print(repr(self.db.file.get(nid, 'content')))
+        print(repr(self.db.file.get(nid, 'binary_content')))
+        p3val='file1 is not text, retrieve using binary_content property. mdsum: 0e1d1b47e4bd1beab3afc9b79f596c1d'
+
+        if sys.version_info[0] > 2:
+            # python 3
+            self.assertEqual(self.db.file.get(nid, 'content'), p3val)
+            self.assertEqual(self.db.file.get(nid, 'binary_content'),
+                             bstr)
+        else:
+            # python 2
+            self.assertEqual(self.db.file.get(nid, 'content'), bstr)
+            self.assertEqual(self.db.file.get(nid, 'binary_content'), bstr)
+
     # Link
     def testLinkChange(self):
         self.assertRaises(IndexError, self.db.issue.create, title="spam",