Enable timing stats reporting in REST interface.

Author: rouilj

CHANGES.txt

@@ -43,7 +43,9 @@ Features:
   (John Rouillard)
 - New filter command defined in roundup-admin. (Partial fix for
   issue724648.) (John Rouillard)
-
+- New parameter @stats for REST interface that provides the same
+  performance stats as the web interface's CGI_SHOW_TIMING env
+  variable. (John Rouillard)
 
 2020-04-05 2.0.0 beta 0
 

doc/rest.txt

@@ -137,6 +137,33 @@ will be returned. It should be used as a hint that the REST endpoint
 will be going away. See https://tools.ietf.org/html/rfc8594 for
 details on this header and the sunset link type.
 
+Hyperdb Stats
+=============
+
+Adding ``@stats=true`` as a GET query parameter or POST data item will
+augment the response with an ``@stats`` dictionary. Any value other
+than ``true`` (any case) will disable the ``@stats`` dictionary. When
+stats are enabled the response includes an ``@stats`` member and looks
+like::
+
+  { "data": {
+      ...
+      "@stats": {
+            "cache_hits": 3,
+            "cache_misses": 1,
+            "get_items": 0.0009722709655761719,
+            "filtering": 0,
+            "elapsed": 0.04731464385986328
+      }
+    }
+  }
+
+These are the same values returned in the html interface by setting
+the ``CGI_SHOW_TIMING`` environment variable. By default performance
+stats are not shown. The fields are subject to change. An
+understanding of the code is recommended if you are going to use this
+info.
+
 Versioning
 ==========
 

roundup/rest.py

@@ -116,6 +116,9 @@ def format_object(self, *args, **kwargs):
                 }
             }
         else:
+            if hasattr(self.db, 'stats') and self.report_stats:
+                self.db.stats['elapsed'] = time.time()-self.start
+                data['@stats'] = self.db.stats
             result = {
                 'data': data
             }
@@ -370,6 +373,11 @@ def __init__(self, client, db):
         self.client = client
         self.db = db
         self.translator = client.translator
+        # record start time for statistics reporting
+        self.start = time.time()
+        # disable stat reporting by default enable with @stats=True
+        # query param
+        self.report_stats = False
         # This used to be initialized from client.instance.actions which
         # would include too many actions that do not make sense in the
         # REST-API context, so for now we only permit the retire and
@@ -1951,6 +1959,14 @@ def dispatch(self, method, uri, input):
         except (KeyError, TypeError):
             pretty_output = True
 
+        # check for runtime statistics
+        try:
+            self.report_stats = input['@stats'].value.lower() == "true"
+        # Can also return a TypeError ("not indexable")
+        # In case the FieldStorage could not parse the result
+        except (KeyError, TypeError):
+            report_stats = False
+
         # check for @apiver in query string
         msg = ("Unrecognized version: %s. "
                "See /rest without specifying version "

test/rest_common.py

@@ -35,7 +35,7 @@ def dst(self, dt):
 from roundup.rest import RestfulInstance, calculate_etag
 from roundup.backends import list_backends
 from roundup.cgi import client
-from roundup.anypy.strings import b2s, s2b
+from roundup.anypy.strings import b2s, s2b, us2u
 import random
 
 from roundup.backends.sessions_dbm import OneTimeKeys
@@ -1319,6 +1319,107 @@ def testDispatchPost(self):
            "http://tracker.example/cgi-bin/roundup.cgi/bugs/rest/data/user/2")
 
 
+    def testStatsGen(self):
+        # check stats being returned by put and get ops
+        # using dispatch which parses the @stats query param
+
+        # find correct py2/py3 list comparison ignoring order
+        try:
+            list_test = self.assertCountEqual  # py3
+        except AttributeError:
+            list_test = self.assertItemsEqual  # py2.7+
+
+        # get stats
+        form = cgi.FieldStorage()
+        form.list = [
+            cgi.MiniFieldStorage('@stats', 'True'),
+        ]
+        results = self.server.dispatch('GET',
+                 "/rest/data/user/1/realname",
+                                 form)
+        self.assertEqual(self.dummy_client.response_code, 200)
+        json_dict = json.loads(b2s(results))
+
+        # check that @stats are defined
+        self.assertTrue( '@stats' in json_dict['data'] )
+        # check that the keys are present
+        # not validating values as that changes
+        valid_fields= [ us2u('elapsed'),
+                        us2u('cache_hits'),
+                        us2u('cache_misses'),
+                        us2u('get_items'),
+                        us2u('filtering') ]
+        list_test(valid_fields,json_dict['data']['@stats'].keys())
+
+        # Make sure false value works to suppress @stats
+        form = cgi.FieldStorage()
+        form.list = [
+            cgi.MiniFieldStorage('@stats', 'False'),
+        ]
+        results = self.server.dispatch('GET',
+                 "/rest/data/user/1/realname",
+                                 form)
+        self.assertEqual(self.dummy_client.response_code, 200)
+        json_dict = json.loads(b2s(results))
+        print(results)
+        # check that @stats are not defined
+        self.assertTrue( '@stats' not in json_dict['data'] )
+
+        # Make sure non-true value works to suppress @stats
+        # false will always work
+        form = cgi.FieldStorage()
+        form.list = [
+            cgi.MiniFieldStorage('@stats', 'random'),
+        ]
+        results = self.server.dispatch('GET',
+                 "/rest/data/user/1/realname",
+                                 form)
+        self.assertEqual(self.dummy_client.response_code, 200)
+        json_dict = json.loads(b2s(results))
+        print(results)
+        # check that @stats are not defined
+        self.assertTrue( '@stats' not in json_dict['data'] )
+
+        # if @stats is not defined there should be no stats
+        results = self.server.dispatch('GET',
+                 "/rest/data/user/1/realname",
+                                 self.empty_form)
+        self.assertEqual(self.dummy_client.response_code, 200)
+        json_dict = json.loads(b2s(results))
+
+        # check that @stats are not defined
+        self.assertTrue( '@stats' not in json_dict['data'] )
+
+
+
+        # change admin's realname via a normal web form
+        # This generates a FieldStorage that looks like:
+        #  FieldStorage(None, None, [])
+        # use etag from header
+        #
+        # Also use GET on the uri via the dispatch to retrieve
+        # the results from the db.
+        etag = calculate_etag(self.db.user.getnode('1'),
+                              self.db.config['WEB_SECRET_KEY'])
+        headers={"if-match": etag,
+                 "accept": "application/vnd.json.test-v1+json",
+        }
+        form = cgi.FieldStorage()
+        form.list = [
+            cgi.MiniFieldStorage('data', 'Joe Doe'),
+            cgi.MiniFieldStorage('@apiver', '1'),
+            cgi.MiniFieldStorage('@stats', 'true'),
+        ]
+        self.headers = headers
+        self.server.client.request.headers.get = self.get_header
+        self.db.setCurrentUser('admin') # must be admin to change user
+        results = self.server.dispatch('PUT',
+                            "/rest/data/user/1/realname",
+                            form)
+        self.assertEqual(self.dummy_client.response_code, 200)
+        json_dict = json.loads(b2s(results))
+        list_test(valid_fields,json_dict['data']['@stats'].keys())
+
     def testDispatch(self):
         """
         run changes through rest dispatch(). This also tests