Merge in non-conflicting changes from ba67e397f063

including workaround for:
   https://bugs.python.org/issue27777

1) cgi/client.py: override cgi.FieldStorage's make_file so that file
   is always created in binary/byte mode. This means that json (and
   xml) are bytes not strings.
2) rest.py: try harder to find dicttoxml in roundup directory or on
   sys.path. This just worked under python 2 but python 3 only
   searches sys.path by default and does not search relative like
   python 2.
3) test/rest_common.py: workaround for issue27777

Also removed an unneeded case insensitive dict implementation.

Author: rouilj

roundup/cgi/client.py

@@ -375,7 +375,19 @@ def __init__(self, instance, request, env, form=None, translator=None):
                     logger.debug("Setting CONTENT_LENGTH to 0 for method: %s",
                                 self.env['REQUEST_METHOD'])
 
+            # cgi.FieldStorage must save all data as
+            # binary/bytes. This is needed for handling json and xml
+            # data blobs under python 3. Under python 2, str and binary
+            # are interchangable, not so under 3.
+            def make_file(self):
+                ''' work around https://bugs.python.org/issue27777 '''
+                import tempfile
+                return tempfile.TemporaryFile("wb+")
+
+            saved_make_file = cgi.FieldStorage.make_file
+            cgi.FieldStorage.make_file = make_file
             self.form = cgi.FieldStorage(fp=request.rfile, environ=env)
+            cgi.FieldStorage.make_file = saved_make_file
             # In some case (e.g. content-type application/xml), cgi
             # will not parse anything. Fake a list property in this case
             if self.form.list is None:

roundup/rest.py

@@ -20,14 +20,20 @@
 import re
 
 try:
-    from dicttoxml import dicttoxml
+    # if dicttoxml installed in roundup directory, use it
+    from .dicttoxml import dicttoxml
 except ImportError:
-    dicttoxml = None
+    try:
+        # else look in sys.path
+        from dicttoxml import dicttoxml
+    except ImportError:
+        # else not supported
+        dicttoxml = None
 
 from roundup import hyperdb
 from roundup import date
 from roundup import actions
-from roundup.anypy.strings import bs2b, s2b
+from roundup.anypy.strings import bs2b, b2s
 from roundup.exceptions import *
 from roundup.cgi.exceptions import *
 
@@ -1346,12 +1352,12 @@ def dispatch(self, method, uri, input):
             "Access-Control-Allow-Methods",
             "HEAD, OPTIONS, GET, PUT, DELETE, PATCH"
         )
-
         # Is there an input.value with format json data?
         # If so turn it into an object that emulates enough
         # of the FieldStorge methods/props to allow a response.
         content_type_header = headers.get('Content-Type', None)
-        if type(input.value) == str and content_type_header:
+        # python2 is str type, python3 is bytes
+        if type(input.value) in ( str, bytes ) and content_type_header:
             parsed_content_type_header = content_type_header
             # the structure of a content-type header
             # is complex: mime-type; options(charset ...)
@@ -1366,7 +1372,7 @@ def dispatch(self, method, uri, input):
             # for example.
             if content_type_header.lower() == "application/json":
                 try:
-                    input = SimulateFieldStorageFromJson(input.value)
+                    input = SimulateFieldStorageFromJson(b2s(input.value))
                 except ValueError as msg:
                     output = self.error_obj(400, msg)
 
@@ -1404,7 +1410,7 @@ def dispatch(self, method, uri, input):
 
         # Make output json end in a newline to
         # separate from following text in logs etc..
-        return s2b(output + "\n")
+        return bs2b(output + "\n")
 
 
 class RoundupJSONEncoder(json.JSONEncoder):

test/rest_common.py

@@ -85,7 +85,7 @@ def tearDown(self):
     def get_header (self, header, not_found=None):
         try:
             return self.headers[header.lower()]
-        except (AttributeError, KeyError):
+        except (AttributeError, KeyError, TypeError):
             return not_found
 
     def testGet(self):
@@ -352,7 +352,7 @@ def testEtagProcessing(self):
 
         Run over header only, etag in form only, both,
         each one broke and no etag. Use the put command
-        to triger the etag checking code.
+        to trigger the etag checking code.
         '''
         for mode in ('header', 'etag', 'both',
                      'brokenheader', 'brokenetag', 'none'):
@@ -399,14 +399,27 @@ def testEtagProcessing(self):
             else:
                 self.assertEqual(self.dummy_client.response_code, 412)
 
+    def make_file(self, arg=None):
+        ''' work around https://bugs.python.org/issue27777 '''
+        import tempfile
+        return tempfile.TemporaryFile("wb+")
+
     def testDispatch(self):
         """
         run changes through rest dispatch(). This also tests
         sending json payload through code as dispatch is the
         code that changes json payload into something we can
         process.
         """
-        # Set joe's 'realname' using json data.
+
+        # Override the make_file so it is always set to binary
+        # read mode. This is needed so we can send a json 
+        # body.
+        saved_make_file = cgi.FieldStorage.make_file
+        cgi.FieldStorage.make_file = self.make_file
+
+        # TEST #1
+        # PUT: joe's 'realname' using json data.
         # simulate: /rest/data/user/<id>/realname
         # use etag in header
         etag = calculate_etag(self.db.user.getnode(self.joeid))
@@ -417,6 +430,7 @@ def testDispatch(self):
         }
         headers={"accept": "application/json",
                  "content-type": env['CONTENT_TYPE'],
+                 "content-length": env['CONTENT_LENGTH'],
                  "etag": etag
         }
         self.headers=headers
@@ -439,24 +453,29 @@ def testDispatch(self):
                          'Joe Doe 1')
         del(self.headers)
 
+        # TEST #2
         # Set joe's 'realname' using json data.
         # simulate: /rest/data/user/<id>/realname
         # use etag in payload
         etag = calculate_etag(self.db.user.getnode(self.joeid))
         body=s2b('{ "@etag": "%s", "data": "Joe Doe 2" }'%etag)
         env = { "CONTENT_TYPE": "application/json",
                 "CONTENT_LENGTH": len(body),
-                "REQUEST_METHOD": "PUT"
+                "REQUEST_METHOD": "PUT",
         }
-        headers={"accept": "application/json",
-                 "content-type": env['CONTENT_TYPE']
-        }
-        self.headers=headers
+        self.headers=None  # have FieldStorage get len from env.
         body_file=BytesIO(body)  # FieldStorage needs a file
         form = cgi.FieldStorage(body_file,
-                                headers=headers,
+                                headers=None,
                                 environ=env)
         self.server.client.request.headers.get=self.get_header
+
+        headers={"accept": "application/json",
+                 "content-type": env['CONTENT_TYPE'],
+                 "etag": etag
+        }
+        self.headers=headers # set for dispatch
+
         results = self.server.dispatch('PUT',
                             "/rest/data/user/%s/realname"%self.joeid,
                             form)
@@ -468,12 +487,13 @@ def testDispatch(self):
                          'Joe Doe 2')
         del(self.headers)
 
+        # TEST #3
         # change Joe's realname via a normal web form
         # This generates a FieldStorage that looks like:
         #  FieldStorage(None, None, [])
         # use etag from header
         #
-        # also use a GET on the uri via the dispatch to get
+        # Also use GET on the uri via the dispatch to retrieve
         # the results from the db.
         etag = calculate_etag(self.db.user.getnode(self.joeid))
         headers={"etag": etag,
@@ -499,12 +519,14 @@ def testDispatch(self):
         self.assertEqual(json_dict['data']['link'],
                          "http://tracker.example/cgi-bin/"
                          "roundup.cgi/bugs/rest/data/user/3/realname") 
-        self.assertEqual(json_dict['data']['type'], "<type 'str'>")
+        self.assertIn(json_dict['data']['type'], ("<class 'str'>",
+                                                  "<type 'str'>"))
         self.assertEqual(json_dict['data']["id"], "3")
         del(self.headers)
 
 
-        # PATCH joe's email address with json
+        # TEST #4
+        # PATCH: joe's email address with json
         # save address so we can use it later
         stored_results = self.server.get_element('user', self.joeid,
                                                  self.empty_form)
@@ -517,7 +539,8 @@ def testDispatch(self):
                 "REQUEST_METHOD": "PATCH"
         }
         headers={"accept": "application/json",
-                 "content-type": env['CONTENT_TYPE']
+                 "content-type": env['CONTENT_TYPE'],
+                 "content-length": len(body)
         }
         self.headers=headers
         body_file=BytesIO(body)  # FieldStorage needs a file
@@ -535,7 +558,7 @@ def testDispatch(self):
         self.assertEqual(results['data']['attributes']['address'],
                          '[email protected]')
 
-        # and set it back
+        # and set it back reusing env and headers from last test
         etag = calculate_etag(self.db.user.getnode(self.joeid))
         body=s2b('{ "address": "%s", "@etag": "%s"}'%(
             stored_results['data']['attributes']['address'],
@@ -557,15 +580,21 @@ def testDispatch(self):
                          '[email protected]')
         del(self.headers)
 
-        # POST to create new issue
+        # TEST #5
+        # POST: create new issue
+        # no etag needed
+        # FIXME at some point we probably want to implement
+        # Post Once Only, so we need to add a Post Once Exactly
+        # test and a resubmit as well.
+        etag = "not needed"
         body=b'{ "title": "foo bar", "priority": "critical" }'
-
         env = { "CONTENT_TYPE": "application/json",
                 "CONTENT_LENGTH": len(body),
                 "REQUEST_METHOD": "POST"
         }
         headers={"accept": "application/json",
-                 "content-type": env['CONTENT_TYPE']
+                 "content-type": env['CONTENT_TYPE'],
+                 "content-length": len(body)
         }
         self.headers=headers
         body_file=BytesIO(body)  # FieldStorage needs a file
@@ -588,6 +617,9 @@ def testDispatch(self):
                          'foo bar')
         del(self.headers)
 
+        # reset the make_file method in the class
+        cgi.FieldStorage.make_file = saved_make_file
+
     def testPut(self):
         """
         Change joe's 'realname'