Fix string/bytes issues under python 3.

1) cgi/client.py: override cgi.FieldStorage's make_file so that file
   is always created in binary/byte mode. This means that json (and
   xml) are bytes not strings.
2) rest.py: try harder to find dicttoxml in roundup directory or on
   sys.path. This just worked under python 2 but python 3 only
   searches sys.path by default and does not search relative like
   python 2.
3) rest.py: replace headers.getheader call removed from python 3 with
   equivalent code.
4) rest.py: make value returned from dispatch into bytes not string.
5) test/caseinsensitivedict.py, test/test_CaseInsensitiveDict.py:
   get code from stackoverflow that implements a case insensitive key
   dict. So dict['foo'], dict['Foo'] are the same entry. Used for
   looking up headers in mocked http rewuset header array.
6) test/rest_common.py: rework tests for etags and rest to properly
   supply bytes to the called routines. Calls to s2b and b2s and use
   of BytesIO and overriding make_file in cgi.FieldStorage to try to
   make sure it works under python 3.

Author: rouilj

roundup/cgi/client.py

@@ -375,7 +375,18 @@ def __init__(self, instance, request, env, form=None, translator=None):
                     logger.debug("Setting CONTENT_LENGTH to 0 for method: %s",
                                 self.env['REQUEST_METHOD'])
 
+            # cgi.FieldStorage must save all data as
+            # binary/bytes. This is needed for handling json and xml
+            # data blobs under python 3. Under python 2, str and binary
+            # are interchangable, not so under 3.
+            def make_file(self):
+                import tempfile
+                return tempfile.TemporaryFile("wb+")
+
+            saved_make_file = cgi.FieldStorage.make_file
+            cgi.FieldStorage.make_file = make_file
             self.form = cgi.FieldStorage(fp=request.rfile, environ=env)
+            cgi.FieldStorage.make_file = saved_make_file
             # In some case (e.g. content-type application/xml), cgi
             # will not parse anything. Fake a list property in this case
             if self.form.list is None:

roundup/rest.py

@@ -20,9 +20,15 @@
 import re
 
 try:
-    from dicttoxml import dicttoxml
+    # if dicttoxml installed in roundup directory, use it
+    from .dicttoxml import dicttoxml
 except ImportError:
-    dicttoxml = None
+    try:
+        # else look in sys.path
+        from dicttoxml import dicttoxml
+    except ImportError:
+        # else not supported
+        dicttoxml = None
 
 from roundup import hyperdb
 from roundup import date
@@ -160,7 +166,8 @@ def obtain_etags(headers,input):
     etags = []
     if '@etag' in input:
         etags.append(input['@etag'].value);
-    etags.append(headers.getheader("ETag", None))
+    if "ETag" in headers:
+        etags.append(headers["ETag"])
     return etags
 
 def parse_accept_header(accept):
@@ -220,7 +227,9 @@ def parse_accept_header(accept):
             else:
                 media_params.append((key, value))
         result.append((media_type, dict(media_params), q))
-    result.sort(lambda x, y: -cmp(x[2], y[2]))
+    # was: result.sort(lambda x, y: -cmp(x[2], y[2]))
+    # change for python 3 support
+    result.sort(key=lambda x: x[2], reverse=True)
     return result
 
 
@@ -1299,7 +1308,9 @@ def dispatch(self, method, uri, input):
         headers = self.client.request.headers
         # Never allow GET to be an unsafe operation (i.e. data changing).
         # User must use POST to "tunnel" DELETE, PUT, OPTIONS etc.
-        override = headers.getheader('X-HTTP-Method-Override')
+        override = None
+        if 'X-HTTP-Method-Override' in headers:
+            override = headers['X-HTTP-Method-Override']
         output = None
         if override:
             if method.upper() != 'GET':
@@ -1314,7 +1325,9 @@ def dispatch(self, method, uri, input):
                     uri)
 
         # parse Accept header and get the content type
-        accept_header = parse_accept_header(headers.getheader('Accept'))
+        accept_header = []
+        if 'Accept' in headers:
+            accept_header = parse_accept_header(headers['Accept'])
         accept_type = "invalid"
         for part in accept_header:
             if part[0] in self.__accepted_content_type:
@@ -1350,7 +1363,9 @@ def dispatch(self, method, uri, input):
         # Is there an input.value with format json data?
         # If so turn it into an object that emulates enough
         # of the FieldStorge methods/props to allow a response.
-        content_type_header = headers.getheader('Content-Type', None)
+        content_type_header = None
+        if 'Content-Type' in headers:
+            content_type_header = headers['Content-Type']
         if type(input.value) == str and content_type_header:
             parsed_content_type_header = content_type_header
             # the structure of a content-type header
@@ -1404,7 +1419,7 @@ def dispatch(self, method, uri, input):
 
         # Make output json end in a newline to
         # separate from following text in logs etc..
-        return output + "\n"
+        return bs2b(output + "\n")
 
 
 class RoundupJSONEncoder(json.JSONEncoder):