issue2551203 - Add support for CORS preflight request

Add support for unauthenticated CORS preflight and fix headers for
CORS.

client.py:

  pass through unauthenticated CORS preflight to rest backend. Normal
     rest OPTION handlers (including tracker defined extensions) can
     see and handle the request.

  make some error cases return error json with crrect mime type rather
     than plain text tracebacks.

  create new functions to verify origin and referer that filter using
     allowed origins setting.

  remove tracker base url from error message is referer is not at an
     allowed origin.

rest.py:

  fix up OPTION methods handlers to include
    Access-Control-Allow-Methods that are the same as the Allow
    header.

  set cache to one week for all Access-Control headers for CORS
    preflight only.

  remove self.client.setHeader("Access-Control-Allow-Origin", "*") and
    set Access-Control-Allow-Origin to the client supplied origin if
    it passes allowed origin checks. Required for CORS otherwise data
    isn't available to caller. Set for all responses.


  set Vary header now includes Origin as responses can differ based on
    Origin for all responses.

  set Access-Control-Allow-Credentials to true on all responses.

test_liveserver.py:

  run server with setting to enforce origin csrf header check

  run server with setting to enforce x-requested-with csrf header check

  run server with setting for allowed_api_origins

  requests now set required csrf headers

  test preflight request on collections

  check new headers and Origin is no longer '*'


  rewrite all compression checks to use a single method with argument
     to use different compression methods. Reduce a lot of code
     duplication and makes updating for new headers easier.


test_cgi:

  test new error messages in client.py

  account for new headers

  test preflight and new code paths

Author: rouilj

CHANGES.txt

@@ -99,6 +99,8 @@ Fixed:
   empty password to login.
 - issue2551207 - Fix sorting by order attribute if order attributes can
   be None. Add a test.
+- issue2551203 fix CORS requests by providing proper headers and allowing
+  unauthenticted CORS preflight requests.
 
 Features:
 

roundup/cgi/client.py

@@ -620,7 +620,32 @@ def handle_rest(self):
             self.write(output)
             return
 
-        if not self.db.security.hasPermission('Rest Access', self.userid):
+        # allow preflight request even if unauthenticated
+        if ( self.env['REQUEST_METHOD'] == "OPTIONS"
+             and self.request.headers.get ("Access-Control-Request-Headers")
+             and self.request.headers.get ("Access-Control-Request-Method")
+             and self.request.headers.get ("Origin")
+        ):
+            # verify Origin is allowed
+            if not self.is_origin_header_ok(api=True):
+                # Use code 400 as 401 and 403 imply that authentication
+                # is needed or authenticates person is not authorized.
+                # Preflight doesn't do authentication.
+                self.response_code = 400
+                msg = self._("Client is not allowed to use Rest Interface.")
+                output = s2b(
+                    '{ "error": { "status": 400, "msg": "%s" } }'%msg )
+                self.setHeader("Content-Length", str(len(output)))
+                self.setHeader("Content-Type", "application/json")
+                self.write(output)
+                return
+            
+            # Call rest library to handle the pre-flight request
+            handler = rest.RestfulInstance(self, self.db)
+            output = handler.dispatch(self.env['REQUEST_METHOD'],
+                                      self.path, self.form)
+
+        elif not self.db.security.hasPermission('Rest Access', self.userid):
             self.response_code = 403
             output = s2b('{ "error": { "status": 403, "msg": "Forbidden." } }')
             self.setHeader("Content-Length", str(len(output)))
@@ -636,14 +661,12 @@ def handle_rest(self):
             # raises exception on check failure.
             csrf_ok = self.handle_csrf(api=True)
         except (Unauthorised, UsageError) as msg:
-            # report exception back to server
-            exc_type, exc_value, exc_tb = sys.exc_info()
             # FIXME should return what the client requests
             # via accept header.
-            output = s2b("%s: %s\n"%(exc_type, exc_value))
+            output = s2b('{ "error": { "status": 400, "msg": "%s"}}'% str(msg))
             self.response_code = 400
             self.setHeader("Content-Length", str(len(output)))
-            self.setHeader("Content-Type", "text/plain")
+            self.setHeader("Content-Type", "application/json")
             self.write(output)
             csrf_ok = False # we had an error, failed check
             return
@@ -1223,9 +1246,39 @@ def is_origin_header_ok(self, api=False):
         # Original spec says origin is case sensitive match.
         # Living spec doesn't address Origin value's case or
         # how to compare it. So implement case sensitive.... 
-        if allowed_origins[0] == '*' or origin in allowed_origins:
+        if allowed_origins:
+            if allowed_origins[0] == '*' or origin in allowed_origins:
+                return True
+
+        return False
+
+    def is_referer_header_ok(self, api=False):
+        referer = self.env['HTTP_REFERER']
+        # parse referer and create an origin
+        referer_comp = urllib_.urlparse(referer)
+
+        # self.base always has trailing /, so add trailing / to referer_origin
+        referer_origin = "%s://%s/"%(referer_comp[0], referer_comp[1])
+        foundat = self.base.find(referer_origin)
+        if foundat == 0:
             return True
 
+        if not api:
+            return False
+        
+        allowed_origins = self.db.config['WEB_ALLOWED_API_ORIGINS']
+        if allowed_origins[0] == '*':
+            return True
+
+        # For referer, loop over allowed_api_origins and
+        # see if any of them are a prefix to referer, case sensitive.
+        # Append / to each origin so that:
+        # an allowed_origin of https://my.host does not match
+        # a referer of https://my.host.com/my/path
+        for allowed_origin in allowed_origins:
+            foundat = referer_origin.find(allowed_origin + '/')
+            if foundat == 0:
+                return True
         return False
 
     def handle_csrf(self, api=False):
@@ -1335,13 +1388,11 @@ def handle_csrf(self, api=False):
         # self.base always matches: ^https?://hostname
         enforce=config['WEB_CSRF_ENFORCE_HEADER_REFERER']
         if 'HTTP_REFERER' in self.env and enforce != "no":
-            referer = self.env['HTTP_REFERER']
-            # self.base always has trailing /
-            foundat = referer.find(self.base)
-            if foundat != 0:
+            if not self.is_referer_header_ok(api=api):
+                referer = self.env['HTTP_REFERER']
                 if enforce in ('required', 'yes'):
                     logger.error(self._("csrf Referer header check failed for user%s. Value=%s"), current_user, referer)
-                    raise Unauthorised(self._("Invalid Referer %s, %s")%(referer,self.base))
+                    raise Unauthorised(self._("Invalid Referer: %s")%(referer))
                 elif enforce == 'logfailure':
                     logger.warning(self._("csrf Referer header check failed for user%s. Value=%s"), current_user, referer)
             else:

roundup/rest.py

@@ -1751,6 +1751,10 @@ def options_element(self, class_name, item_id, input):
             "Allow",
             "OPTIONS, GET, PUT, DELETE, PATCH"
         )
+        self.client.setHeader(
+            "Access-Control-Allow-Methods",
+            "OPTIONS, GET, PUT, DELETE, PATCH"
+        )
         return 204, ""
 
     @Routing.route("/data/<:class_name>/<:item_id>/<:attr_name>", 'OPTIONS')
@@ -1774,12 +1778,20 @@ def option_attribute(self, class_name, item_id, attr_name, input):
                 "Allow",
                 "OPTIONS, GET, PUT, DELETE, PATCH"
             )
+            self.client.setHeader(
+                "Access-Control-Allow-Methods",
+                "OPTIONS, GET, PUT, DELETE, PATCH"
+            )
         elif attr_name in class_obj.getprops(protected=True):
             # It must match a protected prop. These can't be written.
             self.client.setHeader(
                 "Allow",
                 "OPTIONS, GET"
             )
+            self.client.setHeader(
+                "Access-Control-Allow-Methods",
+                "OPTIONS, GET"
+            )
         else:
             raise NotFound('Attribute %s not valid for Class %s' % (
                 attr_name, class_name))
@@ -1910,6 +1922,10 @@ def options_describe(self, input):
             "Allow",
             "OPTIONS, GET"
         )
+        self.client.setHeader(
+            "Access-Control-Allow-Methods",
+            "OPTIONS, GET"
+        )
         return 204, ""
 
     @Routing.route("/data")
@@ -1939,6 +1955,10 @@ def options_data(self, input):
             "Allow",
             "OPTIONS, GET"
         )
+        self.client.setHeader(
+            "Access-Control-Allow-Methods",
+            "OPTIONS, GET"
+        )
         return 204, ""
 
     @Routing.route("/summary")
@@ -2161,20 +2181,46 @@ def dispatch(self, method, uri, input):
         # with invalid values.
         data_type = ext_type or accept_type or headers.get('Accept') or "invalid"
 
-        # add access-control-allow-* to support CORS
-        self.client.setHeader("Access-Control-Allow-Origin", "*")
+        if method.upper() == 'OPTIONS':
+            # add access-control-allow-* access-control-max-age to support
+            # CORS preflight
+            self.client.setHeader(
+                "Access-Control-Allow-Headers",
+                "Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override"
+            )
+            # can be overridden by options handlers to provide supported
+            # methods for endpoint
+            self.client.setHeader(
+                "Access-Control-Allow-Methods",
+                "HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH"
+            )
+            # cache the Access headings for a week. Allows one CORS pre-flight
+            # request to be reused again and again.
+            self.client.setHeader("Access-Control-Max-Age", "86400")
+
+        # response may change based on Origin value.
+        self.client.setVary("Origin") 
+
+        # Allow-Origin must match origin supplied by client. '*' doesn't
+        # work for authenticated requests.
         self.client.setHeader(
-            "Access-Control-Allow-Headers",
-            "Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override"
+            "Access-Control-Allow-Origin",
+            self.client.request.headers.get("Origin")
         )
+
+        # allow credentials
         self.client.setHeader(
-            "Allow",
-            "OPTIONS, GET, POST, PUT, DELETE, PATCH"
+            "Access-Control-Allow-Credentials",
+            "true"
         )
+        # set allow header in case of error. 405 handlers below should
+        # replace it with a custom version as will OPTIONS handler
+        # doing CORS.
         self.client.setHeader(
-            "Access-Control-Allow-Methods",
-            "HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH"
+            "Allow",
+            "OPTIONS, GET, POST, PUT, DELETE, PATCH"
         )
+
         # Is there an input.value with format json data?
         # If so turn it into an object that emulates enough
         # of the FieldStorge methods/props to allow a response.

test/test_cgi.py

@@ -1169,6 +1169,20 @@ def hasPermission(s, p, classname=None, d=None, e=None, **kw):
         del(cl.env['HTTP_ORIGIN'])
         cl.db.config.WEB_ALLOWED_API_ORIGINS = ""
         del(out[0])
+
+        # test by setting allowed api origins to *
+        # this should not redirect as it is not an API call.
+        cl.db.config.WEB_ALLOWED_API_ORIGINS = "  *  "
+        cl.env['HTTP_ORIGIN'] = 'http://whoami.com'
+        cl.env['HTTP_REFERER'] = 'https://baz.edu/path/'
+        cl.inner_main()
+        match_at=out[0].find('Invalid Referer: https://baz.edu/path/')
+        print("result of subtest invalid referer:", out[0])
+        self.assertEqual(match_at, 36)
+        del(cl.env['HTTP_ORIGIN'])
+        del(cl.env['HTTP_REFERER'])
+        cl.db.config.WEB_ALLOWED_API_ORIGINS = ""
+        del(out[0])
 
         # clean up from email log
         if os.path.exists(SENDMAILDEBUG):
@@ -1215,7 +1229,7 @@ def wh(s):
         # Should return explanation because content type is text/plain
         # and not text/xml
         cl.handle_rest()
-        self.assertEqual(b2s(out[0]), "<class 'roundup.exceptions.UsageError'>: Required Header Missing\n")
+        self.assertEqual(b2s(out[0]), '{ "error": { "status": 400, "msg": "Required Header Missing"}}')
         del(out[0])
 
         cl = client.Client(self.instance, None,
@@ -1320,7 +1334,7 @@ def wh(s):
         # Should return explanation because content type is text/plain
         # and not text/xml
         cl.handle_rest()
-        self.assertEqual(b2s(out[0]), "<class 'roundup.exceptions.Unauthorised'>: Invalid Origin httxs://bar.edu\n")
+        self.assertEqual(b2s(out[0]), '{ "error": { "status": 400, "msg": "Invalid Origin httxs://bar.edu"}}')
         del(out[0])
 
 
@@ -1332,7 +1346,7 @@ def wh(s):
                             'HTTP_ORIGIN': 'httxs://bar.edu',
                             'HTTP_X_REQUESTED_WITH': 'rest',
                             'HTTP_AUTHORIZATION': 'Basic YWRtaW46YWRtaW4=',
-                            'HTTP_REFERER': 'http://whoami.com/path/',
+                            'HTTP_REFERER': 'httxp://bar.edu/path/',
                             'HTTP_ACCEPT': "application/json;version=1"
                         }, form)
         cl.db = self.db
@@ -1346,11 +1360,68 @@ def wh(s):
 
         cl.write = wh # capture output
 
-        # create third issue
+        # create fourth issue
         cl.handle_rest()
         self.assertIn('"id": "3"', b2s(out[0]))
         del(out[0])
 
+        cl.db.config.WEB_ALLOWED_API_ORIGINS = "httxs://bar.foo.edu httxs://bar.edu"
+        for referer in [ 'httxs://bar.edu/path/foo',
+                         'httxs://bar.edu/path/foo?g=zz',
+                         'httxs://bar.edu']:
+            cl = client.Client(self.instance, None,
+                               {'REQUEST_METHOD':'POST',
+                                'PATH_INFO':'rest/data/issue',
+                                'CONTENT_TYPE': 'application/x-www-form-urlencoded',
+                                'HTTP_ORIGIN': 'httxs://bar.edu',
+                                'HTTP_X_REQUESTED_WITH': 'rest',
+                                'HTTP_AUTHORIZATION': 'Basic YWRtaW46YWRtaW4=',
+                                'HTTP_REFERER': referer,
+                                'HTTP_ACCEPT': "application/json;version=1"
+                               }, form)
+            cl.db = self.db
+            cl.base = 'http://whoami.com/path/'
+            cl._socket_op = lambda *x : True
+            cl._error_message = []
+            cl.request = MockNull()
+            h = { 'content-type': 'application/json',
+                  'accept': 'application/json' }
+            cl.request.headers = MockNull(**h)
+            
+            cl.write = wh # capture output
+
+            # create fourth issue
+            cl.handle_rest()
+            self.assertIn('"id": "', b2s(out[0]))
+            del(out[0])
+        
+        cl.db.config.WEB_ALLOWED_API_ORIGINS = "httxs://bar.foo.edu httxs://bar.edu"
+        cl = client.Client(self.instance, None,
+                           {'REQUEST_METHOD':'POST',
+                            'PATH_INFO':'rest/data/issue',
+                            'CONTENT_TYPE': 'application/x-www-form-urlencoded',
+                            'HTTP_ORIGIN': 'httxs://bar.edu',
+                            'HTTP_X_REQUESTED_WITH': 'rest',
+                            'HTTP_AUTHORIZATION': 'Basic YWRtaW46YWRtaW4=',
+                            'HTTP_REFERER': 'httxp://bar.edu/path/',
+                            'HTTP_ACCEPT': "application/json;version=1"
+                        }, form)
+        cl.db = self.db
+        cl.base = 'http://whoami.com/path/'
+        cl._socket_op = lambda *x : True
+        cl._error_message = []
+        cl.request = MockNull()
+        h = { 'content-type': 'application/json',
+              'accept': 'application/json' }
+        cl.request.headers = MockNull(**h)
+                                      
+        cl.write = wh # capture output
+
+        # create fourth issue
+        cl.handle_rest()
+        self.assertEqual(b2s(out[0]), '{ "error": { "status": 400, "msg": "Invalid Referer: httxp://bar.edu/path/"}}')
+        del(out[0])
+
     def testXmlrpcCsrfProtection(self):
         # set the password for admin so we can log in.
         passwd=password.Password('admin')