Rest rate limiting code first commit. It is a bit rough and turned off

by default.

The current code is lossy. If client connections are fast enough, the
rate limiting code doesn't count every connection. So the client can
get more connections than configured if they are fast enough.

5-20% of the connections are not recorded.

Author: rouilj

roundup/backends/sessions_dbm.py

@@ -129,7 +129,26 @@ def opendb(self, mode):
 
         # open the database with the correct module
         dbm = __import__(db_type)
-        return dbm.open(path, mode)
+
+        retries_left=15
+        while True:
+            try:
+                handle = dbm.open(path, mode)
+                break
+            except OSError as e:
+                # Primarily we want to catch and retry:
+                #   [Errno 11] Resource temporarily unavailable retry
+                # FIXME: make this more specific
+                if retries_left < 0:
+                    # We have used up the retries. Reraise the exception
+                    # that got us here.
+                    raise
+                else:
+                    # delay retry a bit
+                    time.sleep(0.01)
+                    retries_left = retries_left -1
+                    continue  # the while loop
+        return handle
 
     def commit(self):
         pass

roundup/configuration.py

@@ -760,6 +760,20 @@ def str2value(self, value):
 additional REST-API parameters after the roundup web url configured in
 the tracker section. If this variable is set to 'no', the rest path has
 no special meaning and will yield an error message."""),
+        (IntegerNumberOption, 'api_calls_per_interval', "0",
+         "Limit API calls per api_interval_in_sec seconds to\n"
+         "this number.\n"         
+         "Determines the burst rate and the rate that new api\n"
+         "calls will be made available. If set to 360 and\n"
+         "api_intervals_in_sec is set to 3600, the 361st call in\n"
+         "10 seconds results in a 429 error to the caller. It\n"
+         "tells them to wait 10 seconds (360/3600) before making\n"
+         "another api request. A value of 0 turns off rate\n"
+         "limiting in the API. Tune this as needed. See rest\n"
+         "documentation for more info.\n"),
+        (IntegerNumberOption, 'api_interval_in_sec', "3600",
+         "Defines the interval in seconds over which an api client can\n"
+         "make api_calls_per_interval api calls. Tune this as needed.\n"),
         (CsrfSettingOption, 'csrf_enforce_token', "yes",
             """How do we deal with @csrf fields in posted forms.
 Set this to 'required' to block the post and notify

roundup/rest.py

@@ -33,11 +33,14 @@
 from roundup import hyperdb
 from roundup import date
 from roundup import actions
+from roundup.i18n import _
 from roundup.anypy.strings import bs2b, b2s, u2s, is_us
+from roundup.rate_limit import RateLimit, Gcra
 from roundup.exceptions import *
 from roundup.cgi.exceptions import *
 
 import hmac
+from datetime import timedelta
 
 # Py3 compatible basestring
 try:
@@ -1635,14 +1638,80 @@ def summary(self, input):
 
         return 200, result
 
+    def getRateLimit(self):
+        ''' By default set one rate limit for all users. Values
+            for period (in seconds) and count set in config.
+            However there is no reason these settings couldn't
+            be pulled from the user's entry in the database. So define
+            this method to allow a user to change it in the interfaces.py
+            to use a field in the user object.
+        '''
+        # FIXME verify can override from interfaces.py.
+        calls = self.db.config.WEB_API_CALLS_PER_INTERVAL
+        interval = self.db.config.WEB_API_INTERVAL_IN_SEC
+        if calls and interval:
+            return RateLimit(calls,timedelta(seconds=interval))
+        else:
+            # disable rate limiting if either parameter is 0
+            return None
+
     def dispatch(self, method, uri, input):
         """format and process the request"""
+        output = None
+
+        # Before we do anything has the user hit the rate limit.
+        # This should (but doesn't at the moment) bypass
+        # all other processing to minimize load of badly
+        # behaving client.
+
+        # Get the limit here and not in the init() routine to allow
+        # for a different rate limit per user.
+        apiRateLimit = self.getRateLimit()
+
+        if apiRateLimit:  # if None, disable rate limiting
+            gcra=Gcra()
+            # unique key is an "ApiLimit-" prefix and the uid)
+            apiLimitKey = "ApiLimit-%s"%self.db.getuid()
+            otk=self.db.Otk
+            try:
+                val=otk.getall(apiLimitKey)
+                gcra.set_tat_as_string(apiLimitKey, val['tat'])
+            except KeyError:
+                # ignore if tat not set, it's 1970-1-1 by default.
+                pass
+            # see if rate limit exceeded and we need to reject the attempt
+            reject=gcra.update(apiLimitKey, apiRateLimit)
+
+            # Calculate a timestamp that will make OTK expire the
+            # unused entry 1 hour in the future
+            ts = time.time() - (60 * 60 * 24 * 7) + 3600
+            otk.set(apiLimitKey, tat=gcra.get_tat_as_string(apiLimitKey),
+                    __timestamp=ts)
+            otk.commit()
+
+            limitStatus=gcra.status(apiLimitKey, apiRateLimit)
+            if reject:
+                for header, value in limitStatus.items():
+                    self.client.setHeader(header, value)
+                    # User exceeded limits: tell humans how long to wait
+                    # Headers above will do the right thing for api
+                    # aware clients.
+                    msg=_("Api rate limits exceeded. Please wait: %d seconds.")%limitStatus['Retry-After']
+                    output = self.error_obj(429, msg, source="ApiRateLimiter")
+            else:
+                for header,value in limitStatus.items():
+                    # Retry-After will be 0 because
+                    # user still has quota available.
+                    # Don't put out the header.
+                    if header in ( 'Retry-After', ):
+                        continue
+                    self.client.setHeader(header, value)
+
         # if X-HTTP-Method-Override is set, follow the override method
         headers = self.client.request.headers
         # Never allow GET to be an unsafe operation (i.e. data changing).
         # User must use POST to "tunnel" DELETE, PUT, OPTIONS etc.
         override = headers.get('X-HTTP-Method-Override')
-        output = None
         if override:
             if method.upper() == 'POST':
                 logger.debug(

test/rest_common.py

@@ -3,6 +3,9 @@
 import shutil
 import errno
 
+from time import sleep
+from datetime import datetime
+
 from roundup.cgi.exceptions import *
 from roundup.hyperdb import HyperdbValueError
 from roundup.exceptions import *
@@ -621,6 +624,111 @@ def testPagination(self):
         #   page_size < 0
         #   page_index < 0
 
+    def testRestRateLimit(self):
+
+        self.db.config['WEB_API_CALLS_PER_INTERVAL'] = 20
+        self.db.config['WEB_API_INTERVAL_IN_SEC'] = 60
+
+        print("Now realtime start:", datetime.utcnow())
+        # don't set an accept header; json should be the default
+        # use up all our allowed api calls
+        for i in range(20):
+            # i is 0 ... 19
+            self.client_error_message = []
+            results = self.server.dispatch('GET',
+                            "/rest/data/user/%s/realname"%self.joeid,
+                            self.empty_form)
+ 
+            # is successful
+            self.assertEqual(self.server.client.response_code, 200)
+            # does not have Retry-After header as we have
+            # suceeded with this query
+            self.assertFalse("Retry-After" in
+                             self.server.client.additional_headers) 
+            # remaining count is correct
+            self.assertEqual(
+                self.server.client.additional_headers["X-RateLimit-Remaining"],
+                self.db.config['WEB_API_CALLS_PER_INTERVAL'] -1 - i
+                )
+
+        # trip limit
+        self.server.client.additional_headers.clear()
+        results = self.server.dispatch('GET',
+                     "/rest/data/user/%s/realname"%self.joeid,
+                            self.empty_form)
+        print(results)
+        self.assertEqual(self.server.client.response_code, 429)
+
+        self.assertEqual(
+            self.server.client.additional_headers["X-RateLimit-Limit"],
+            self.db.config['WEB_API_CALLS_PER_INTERVAL'])
+        self.assertEqual(
+            self.server.client.additional_headers["X-RateLimit-Limit-Period"],
+            self.db.config['WEB_API_INTERVAL_IN_SEC'])
+        self.assertEqual(
+            self.server.client.additional_headers["X-RateLimit-Remaining"],
+            0)
+        # value will be almost 60. Allow 1-2 seconds for all 20 rounds.
+        self.assertAlmostEqual(
+            self.server.client.additional_headers["X-RateLimit-Reset"],
+            59, delta=1)
+        self.assertEqual(
+            str(self.server.client.additional_headers["Retry-After"]),
+            "3.0")  # check as string
+
+        print("Reset:", self.server.client.additional_headers["X-RateLimit-Reset"])
+        print("Now realtime pre-sleep:", datetime.utcnow())
+        sleep(3.1) # sleep as requested so we can do another login
+        print("Now realtime post-sleep:", datetime.utcnow())
+
+        # this should succeed
+        self.server.client.additional_headers.clear()
+        results = self.server.dispatch('GET',
+                     "/rest/data/user/%s/realname"%self.joeid,
+                            self.empty_form)
+        print(results)
+        print("Reset:", self.server.client.additional_headers["X-RateLimit-Reset-date"])
+        print("Now realtime:", datetime.utcnow())
+        print("Now ts header:", self.server.client.additional_headers["Now"])
+        print("Now date header:", self.server.client.additional_headers["Now-date"])
+
+        self.assertEqual(self.server.client.response_code, 200)
+
+        self.assertEqual(
+            self.server.client.additional_headers["X-RateLimit-Limit"],
+            self.db.config['WEB_API_CALLS_PER_INTERVAL'])
+        self.assertEqual(
+            self.server.client.additional_headers["X-RateLimit-Limit-Period"],
+            self.db.config['WEB_API_INTERVAL_IN_SEC'])
+        self.assertEqual(
+            self.server.client.additional_headers["X-RateLimit-Remaining"],
+            0)
+        self.assertFalse("Retry-After" in
+                         self.server.client.additional_headers) 
+        # we still need to wait a minute for everything to clear
+        self.assertAlmostEqual(
+            self.server.client.additional_headers["X-RateLimit-Reset"],
+            59, delta=1)
+
+        # and make sure we need to wait another three seconds
+        # as we consumed the last api call
+        results = self.server.dispatch('GET',
+                     "/rest/data/user/%s/realname"%self.joeid,
+                            self.empty_form)
+
+        self.assertEqual(self.server.client.response_code, 429)
+        self.assertEqual(
+            str(self.server.client.additional_headers["Retry-After"]),
+            "3.0")  # check as string
+
+        json_dict = json.loads(b2s(results))
+        self.assertEqual(json_dict['error']['msg'],
+                         "Api rate limits exceeded. Please wait: 3 seconds.")
+
+        # reset rest params
+        self.db.config['WEB_API_CALLS_PER_INTERVAL'] = 0
+        self.db.config['WEB_API_INTERVAL_IN_SEC'] = 3600
+            
     def testEtagGeneration(self):
         ''' Make sure etag generation is stable