added capturing of stats

Author: Richard Jones

BUILD.txt

@@ -21,14 +21,18 @@ running:
 7.  python setup.py sdist
     (if you find sdist a little verbose, add "--quiet" to the end of the
      command)
-8.  unpack the new dist file in /tmp then a) run_test.py and b) demo.py
+8. unpack the new dist file in /tmp then a) run_test.py and b) demo.py
     with all available Python versions.
+9.  python setup.py bdist_rpm
+10. python setup.py bdist_wininst
 
 So, those commands in a nice, cut'n'pasteable form::
 
  python setup.py clean --all
  python setup.py sdist --manifest-only
  python setup.py sdist --quiet
+ python setup.py bdist_rpm
+ python setup.py bdist_wininst
  python2.3 setup.py register
 
 

roundup/backends/back_anydbm.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_anydbm.py,v 1.141 2004-04-07 01:12:25 richard Exp $
+#$Id: back_anydbm.py,v 1.142 2004-04-25 22:19:15 richard Exp $
 '''This module defines a backend that saves the hyperdatabase in a
 database chosen by anydbm. It is guaranteed to always be available in python
 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
@@ -33,7 +33,7 @@
 except AssertionError:
     print "WARNING: you should upgrade to python 2.1.3"
 
-import whichdb, os, marshal, re, weakref, string, copy
+import whichdb, os, marshal, re, weakref, string, copy, time
 from roundup import hyperdb, date, password, roundupdb, security
 from blobfiles import FileStorage
 from sessions_dbm import Sessions, OneTimeKeys
@@ -74,6 +74,8 @@ def __init__(self, config, journaltag=None):
         self.dir = config.DATABASE
         self.classes = {}
         self.cache = {}         # cache of nodes loaded or created
+        self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
+            'filtering': 0}
         self.dirtynodes = {}    # keep track of the dirty nodes by class
         self.newnodes = {}      # keep track of the new nodes by class
         self.destroyednodes = {}# keep track of the destroyed nodes by class
@@ -314,9 +316,12 @@ def getnode(self, classname, nodeid, db=None, cache=1):
             if __debug__:
                 print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
                     nodeid)
+                self.stats['cache_hits'] += 1
             return cache_dict[nodeid]
 
         if __debug__:
+            self.stats['cache_misses'] += 1
+            start_t = time.time()
             print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
 
         # get from the database and save in the cache
@@ -340,6 +345,9 @@ def getnode(self, classname, nodeid, db=None, cache=1):
         if cache:
             cache_dict[nodeid] = res
 
+        if __debug__:
+            self.stats['get_items'] += (time.time() - start_t)
+
         return res
 
     def destroynode(self, classname, nodeid):
@@ -1579,6 +1587,9 @@ def filter(self, search_matches, filterspec, sort=(None,None),
         is a Multilink, in which case the item's property list must
         match the filterspec list.
         """
+        if __debug__:
+            start_t = time.time()
+
         cn = self.classname
 
         # optimise filterspec
@@ -1826,7 +1837,10 @@ def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
             return cmp(a[0], b[0])
 
         l.sort(sortfun)
-        return [i[0] for i in l]
+        l = [i[0] for i in l]
+        if __debug__:
+            self.db.stats['filtering'] += (time.time() - start_t)
+        return l
 
     def count(self):
         '''Get the number of nodes in this class.

roundup/backends/back_mysql.py

@@ -507,6 +507,9 @@ def filter(self, search_matches, filterspec, sort=(None,None),
         if search_matches == {}:
             return []
 
+        if __debug__:
+            start_t = time.time()
+
         cn = self.classname
 
         timezone = self.db.getUserTimezone()
@@ -687,6 +690,8 @@ def filter(self, search_matches, filterspec, sort=(None,None),
         l = [str(row[0]) for row in l]
 
         if not mlsort:
+            if __debug__:
+                self.db.stats['filtering'] += (time.time() - start_t)
             return l
 
         # ergh. someone wants to sort by a multilink.
@@ -705,7 +710,12 @@ def sortfun(a, b, dir=sortby[i], i=i):
                     return cmp(a[1][i], b[1][i])
             r.sort(sortfun)
             i += 1
-        return [i[0] for i in r]
+        r = [i[0] for i in r]
+
+        if __debug__:
+            self.db.stats['filtering'] += (time.time() - start_t)
+
+        return r
 
 class Class(MysqlClass, rdbms_common.Class):
     pass

roundup/backends/rdbms_common.py

@@ -1,4 +1,4 @@
-# $Id: rdbms_common.py,v 1.93 2004-04-22 22:17:34 richard Exp $
+# $Id: rdbms_common.py,v 1.94 2004-04-25 22:19:15 richard Exp $
 ''' Relational database (SQL) backend common code.
 
 Basics:
@@ -76,6 +76,8 @@ def __init__(self, config, journaltag=None):
         # (classname, nodeid) = row
         self.cache = {}
         self.cache_lru = []
+        self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0,
+            'filtering': 0}
 
         # database lock
         self.lockfile = None
@@ -831,9 +833,15 @@ def getnode(self, classname, nodeid):
             # push us back to the top of the LRU
             self.cache_lru.remove(key)
             self.cache_lru.insert(0, key)
+            if __debug__:
+                self.stats['cache_hits'] += 1
             # return the cached information
             return self.cache[key]
 
+        if __debug__:
+            self.stats['cache_misses'] += 1
+            start_t = time.time()
+
         # figure the columns we're fetching
         cl = self.classes[classname]
         cols, mls = self.determine_columns(cl.properties.items())
@@ -880,6 +888,9 @@ def getnode(self, classname, nodeid):
         if len(self.cache_lru) > ROW_CACHE_SIZE:
             del self.cache[self.cache_lru.pop()]
 
+        if __debug__:
+            self.stats['get_items'] += (time.time() - start_t)
+
         return node
 
     def destroynode(self, classname, nodeid):
@@ -1938,6 +1949,9 @@ def filter(self, search_matches, filterspec, sort=(None,None),
         if search_matches == {}:
             return []
 
+        if __debug__:
+            start_t = time.time()
+
         cn = self.classname
 
         timezone = self.db.getUserTimezone()
@@ -2116,6 +2130,8 @@ def filter(self, search_matches, filterspec, sort=(None,None),
         l = [str(row[0]) for row in l]
 
         if not mlsort:
+            if __debug__:
+                self.db.stats['filtering'] += (time.time() - start_t)
             return l
 
         # ergh. someone wants to sort by a multilink.
@@ -2134,7 +2150,12 @@ def sortfun(a, b, dir=sortby[i], i=i):
                     return cmp(a[1][i], b[1][i])
             r.sort(sortfun)
             i += 1
-        return [i[0] for i in r]
+        r = [i[0] for i in r]
+
+        if __debug__:
+            self.db.stats['filtering'] += (time.time() - start_t)
+
+        return r
 
     def count(self):
         '''Get the number of nodes in this class.

roundup/cgi/client.py

@@ -1,4 +1,4 @@
-# $Id: client.py,v 1.172 2004-04-22 22:16:36 richard Exp $
+# $Id: client.py,v 1.173 2004-04-25 22:19:15 richard Exp $
 
 """WWW request handler (also used in the stand-alone server).
 """
@@ -519,7 +519,14 @@ def renderContext(self):
             result = pt.render(self, None, None, **args)
             self.additional_headers['Content-Type'] = pt.content_type
             if os.environ.get('CGI_SHOW_TIMING', ''):
-                s = '<p>Time elapsed: %fs</p></body>'%(time.time()-self.start)
+                s = '<p>Time elapsed: %fs</p>'%(time.time()-self.start)
+                if hasattr(self.db, 'stats'):
+                    s += '''<p>Cache hits: %(cache_hits)d,
+                        misses %(cache_misses)d.
+                        Loading items: %(get_items)f secs.
+                        Filtering: %(filtering)f secs.
+                        </p>'''%self.db.stats
+                s += '</body>'
                 result = result.replace('</body>', s)
             return result
         except templating.NoTemplate, message:

setup.py

@@ -16,7 +16,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: setup.py,v 1.62 2004-04-18 06:21:20 richard Exp $
+# $Id: setup.py,v 1.63 2004-04-25 22:19:14 richard Exp $
 
 from distutils.core import setup, Extension
 from distutils.util import get_platform
@@ -147,6 +147,7 @@ def check_manifest():
         n = len(manifest)
         print '\n*** SOURCE ERROR: There are files missing (%d/%d found)!'%(
             n-len(err), n)
+        print 'Missing:', '\nMissing: '.join(err)
         sys.exit(1)