Skip to content

Commit f0c1c7c

Browse files
author
Richard Jones
committed
node caching now works, and gives a small boost in performance
As a part of this, I cleaned up the DEBUG output and implemented TRACE output (HYPERDBTRACE='file to trace to') with checkpoints at the start of CGI requests. Run roundup with python -O to skip all the DEBUG/TRACE stuff (using if __debug__ which is compiled out with -O)
1 parent 59ad4ad commit f0c1c7c

File tree

5 files changed

+120
-66
lines changed

5 files changed

+120
-66
lines changed

‎CHANGES.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ Fixed:
4242
(thanks dman)
4343
. fixed some sorting issues that were breaking some unit tests under py2.2
4444
. mailgw test output dir was confusing the init test (but only on 2.2 *shrug*)
45+
. node caching now works, and gives a small boost in performance
4546

4647

4748
2002-03-25 - 0.4.1

‎roundup/backends/back_anydbm.py‎

Lines changed: 69 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
1616
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
1717
#
18-
#$Id: back_anydbm.py,v 1.33 2002-04-24 10:38:26 rochecompaan Exp $
18+
#$Id: back_anydbm.py,v 1.34 2002-05-15 06:21:21 richard Exp $
1919
'''
2020
This module defines a backend that saves the hyperdatabase in a database
2121
chosen by anydbm. It is guaranteed to always be available in python
@@ -73,23 +73,23 @@ def __repr__(self):
7373
def __getattr__(self, classname):
7474
"""A convenient way of calling self.getclass(classname)."""
7575
if self.classes.has_key(classname):
76-
if hyperdb.DEBUG:
77-
print '__getattr__', (self, classname)
76+
if __debug__:
77+
print >>hyperdb.DEBUG, '__getattr__', (self, classname)
7878
return self.classes[classname]
7979
raise AttributeError, classname
8080

8181
def addclass(self, cl):
82-
if hyperdb.DEBUG:
83-
print 'addclass', (self, cl)
82+
if __debug__:
83+
print >>hyperdb.DEBUG, 'addclass', (self, cl)
8484
cn = cl.classname
8585
if self.classes.has_key(cn):
8686
raise ValueError, cn
8787
self.classes[cn] = cl
8888

8989
def getclasses(self):
9090
"""Return a list of the names of all existing classes."""
91-
if hyperdb.DEBUG:
92-
print 'getclasses', (self,)
91+
if __debug__:
92+
print >>hyperdb.DEBUG, 'getclasses', (self,)
9393
l = self.classes.keys()
9494
l.sort()
9595
return l
@@ -99,8 +99,8 @@ def getclass(self, classname):
9999
100100
If 'classname' is not a valid class name, a KeyError is raised.
101101
"""
102-
if hyperdb.DEBUG:
103-
print 'getclass', (self, classname)
102+
if __debug__:
103+
print >>hyperdb.DEBUG, 'getclass', (self, classname)
104104
return self.classes[classname]
105105

106106
#
@@ -109,8 +109,8 @@ def getclass(self, classname):
109109
def clear(self):
110110
'''Delete all database contents
111111
'''
112-
if hyperdb.DEBUG:
113-
print 'clear', (self,)
112+
if __debug__:
113+
print >>hyperdb.DEBUG, 'clear', (self,)
114114
for cn in self.classes.keys():
115115
for dummy in 'nodes', 'journals':
116116
path = os.path.join(self.dir, 'journals.%s'%cn)
@@ -123,16 +123,16 @@ def getclassdb(self, classname, mode='r'):
123123
''' grab a connection to the class db that will be used for
124124
multiple actions
125125
'''
126-
if hyperdb.DEBUG:
127-
print 'getclassdb', (self, classname, mode)
126+
if __debug__:
127+
print >>hyperdb.DEBUG, 'getclassdb', (self, classname, mode)
128128
return self._opendb('nodes.%s'%classname, mode)
129129

130130
def _opendb(self, name, mode):
131131
'''Low-level database opener that gets around anydbm/dbm
132132
eccentricities.
133133
'''
134-
if hyperdb.DEBUG:
135-
print '_opendb', (self, name, mode)
134+
if __debug__:
135+
print >>hyperdb.DEBUG, '_opendb', (self, name, mode)
136136

137137
# determine which DB wrote the class file
138138
db_type = ''
@@ -148,8 +148,8 @@ def _opendb(self, name, mode):
148148

149149
# new database? let anydbm pick the best dbm
150150
if not db_type:
151-
if hyperdb.DEBUG:
152-
print "_opendb anydbm.open(%r, 'n')"%path
151+
if __debug__:
152+
print >>hyperdb.DEBUG, "_opendb anydbm.open(%r, 'n')"%path
153153
return anydbm.open(path, 'n')
154154

155155
# open the database with the correct module
@@ -159,8 +159,9 @@ def _opendb(self, name, mode):
159159
raise hyperdb.DatabaseError, \
160160
"Couldn't open database - the required module '%s'"\
161161
"is not available"%db_type
162-
if hyperdb.DEBUG:
163-
print "_opendb %r.open(%r, %r)"%(db_type, path, mode)
162+
if __debug__:
163+
print >>hyperdb.DEBUG, "_opendb %r.open(%r, %r)"%(db_type, path,
164+
mode)
164165
return dbm.open(path, mode)
165166

166167
def _lockdb(self, name):
@@ -194,17 +195,17 @@ def newid(self, classname):
194195
def addnode(self, classname, nodeid, node):
195196
''' add the specified node to its class's db
196197
'''
197-
if hyperdb.DEBUG:
198-
print 'addnode', (self, classname, nodeid, node)
198+
if __debug__:
199+
print >>hyperdb.DEBUG, 'addnode', (self, classname, nodeid, node)
199200
self.newnodes.setdefault(classname, {})[nodeid] = 1
200201
self.cache.setdefault(classname, {})[nodeid] = node
201202
self.savenode(classname, nodeid, node)
202203

203204
def setnode(self, classname, nodeid, node):
204205
''' change the specified node
205206
'''
206-
if hyperdb.DEBUG:
207-
print 'setnode', (self, classname, nodeid, node)
207+
if __debug__:
208+
print >>hyperdb.DEBUG, 'setnode', (self, classname, nodeid, node)
208209
self.dirtynodes.setdefault(classname, {})[nodeid] = 1
209210

210211
# can't set without having already loaded the node
@@ -214,20 +215,26 @@ def setnode(self, classname, nodeid, node):
214215
def savenode(self, classname, nodeid, node):
215216
''' perform the saving of data specified by the set/addnode
216217
'''
217-
if hyperdb.DEBUG:
218-
print 'savenode', (self, classname, nodeid, node)
218+
if __debug__:
219+
print >>hyperdb.DEBUG, 'savenode', (self, classname, nodeid, node)
219220
self.transactions.append((self._doSaveNode, (classname, nodeid, node)))
220221

221222
def getnode(self, classname, nodeid, db=None, cache=1):
222223
''' get a node from the database
223224
'''
224-
if hyperdb.DEBUG:
225-
print 'getnode', (self, classname, nodeid, db)
225+
if __debug__:
226+
print >>hyperdb.DEBUG, 'getnode', (self, classname, nodeid, db)
226227
if cache:
227228
# try the cache
228-
cache = self.cache.setdefault(classname, {})
229-
if cache.has_key(nodeid):
230-
return cache[nodeid]
229+
cache_dict = self.cache.setdefault(classname, {})
230+
if cache_dict.has_key(nodeid):
231+
if __debug__:
232+
print >>hyperdb.TRACE, 'get %s %s cached'%(classname,
233+
nodeid)
234+
return cache_dict[nodeid]
235+
236+
if __debug__:
237+
print >>hyperdb.TRACE, 'get %s %s'%(classname, nodeid)
231238

232239
# get from the database and save in the cache
233240
if db is None:
@@ -241,21 +248,26 @@ def getnode(self, classname, nodeid, db=None, cache=1):
241248
# reverse the serialisation
242249
res = self.unserialise(classname, res)
243250

244-
# store off in the cache
251+
# store off in the cache dict
245252
if cache:
246-
cache[nodeid] = res
253+
cache_dict[nodeid] = res
247254

248255
return res
249256

250257
def hasnode(self, classname, nodeid, db=None):
251258
''' determine if the database has a given node
252259
'''
253-
if hyperdb.DEBUG:
254-
print 'hasnode', (self, classname, nodeid, db)
260+
if __debug__:
261+
print >>hyperdb.DEBUG, 'hasnode', (self, classname, nodeid, db)
262+
255263
# try the cache
256264
cache = self.cache.setdefault(classname, {})
257265
if cache.has_key(nodeid):
266+
if __debug__:
267+
print >>hyperdb.TRACE, 'has %s %s cached'%(classname, nodeid)
258268
return 1
269+
if __debug__:
270+
print >>hyperdb.TRACE, 'has %s %s'%(classname, nodeid)
259271

260272
# not in the cache - check the database
261273
if db is None:
@@ -264,8 +276,8 @@ def hasnode(self, classname, nodeid, db=None):
264276
return res
265277

266278
def countnodes(self, classname, db=None):
267-
if hyperdb.DEBUG:
268-
print 'countnodes', (self, classname, db)
279+
if __debug__:
280+
print >>hyperdb.DEBUG, 'countnodes', (self, classname, db)
269281
# include the new nodes not saved to the DB yet
270282
count = len(self.newnodes.get(classname, {}))
271283

@@ -276,8 +288,8 @@ def countnodes(self, classname, db=None):
276288
return count
277289

278290
def getnodeids(self, classname, db=None):
279-
if hyperdb.DEBUG:
280-
print 'getnodeids', (self, classname, db)
291+
if __debug__:
292+
print >>hyperdb.DEBUG, 'getnodeids', (self, classname, db)
281293
# start off with the new nodes
282294
res = self.newnodes.get(classname, {}).keys()
283295

@@ -302,16 +314,17 @@ def addjournal(self, classname, nodeid, action, params):
302314
'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
303315
'retire' -- 'params' is None
304316
'''
305-
if hyperdb.DEBUG:
306-
print 'addjournal', (self, classname, nodeid, action, params)
317+
if __debug__:
318+
print >>hyperdb.DEBUG, 'addjournal', (self, classname, nodeid,
319+
action, params)
307320
self.transactions.append((self._doSaveJournal, (classname, nodeid,
308321
action, params)))
309322

310323
def getjournal(self, classname, nodeid):
311324
''' get the journal for id
312325
'''
313-
if hyperdb.DEBUG:
314-
print 'getjournal', (self, classname, nodeid)
326+
if __debug__:
327+
print >>hyperdb.DEBUG, 'getjournal', (self, classname, nodeid)
315328
# attempt to open the journal - in some rare cases, the journal may
316329
# not exist
317330
try:
@@ -330,8 +343,8 @@ def getjournal(self, classname, nodeid):
330343

331344
def pack(self, pack_before):
332345
''' delete all journal entries before 'pack_before' '''
333-
if hyperdb.DEBUG:
334-
print 'packjournal', (self, pack_before)
346+
if __debug__:
347+
print >>hyperdb.DEBUG, 'packjournal', (self, pack_before)
335348

336349
pack_before = pack_before.get_tuple()
337350

@@ -383,8 +396,8 @@ def pack(self, pack_before):
383396
def commit(self):
384397
''' Commit the current transactions.
385398
'''
386-
if hyperdb.DEBUG:
387-
print 'commit', (self,)
399+
if __debug__:
400+
print >>hyperdb.DEBUG, 'commit', (self,)
388401
# TODO: lock the DB
389402

390403
# keep a handle to all the database files opened
@@ -407,8 +420,9 @@ def commit(self):
407420
self.transactions = []
408421

409422
def _doSaveNode(self, classname, nodeid, node):
410-
if hyperdb.DEBUG:
411-
print '_doSaveNode', (self, classname, nodeid, node)
423+
if __debug__:
424+
print >>hyperdb.DEBUG, '_doSaveNode', (self, classname, nodeid,
425+
node)
412426

413427
# get the database handle
414428
db_name = 'nodes.%s'%classname
@@ -429,8 +443,8 @@ def _doSaveJournal(self, classname, nodeid, action, params):
429443
entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
430444
params)
431445

432-
if hyperdb.DEBUG:
433-
print '_doSaveJournal', entry
446+
if __debug__:
447+
print >>hyperdb.DEBUG, '_doSaveJournal', entry
434448

435449
# get the database handle
436450
db_name = 'journals.%s'%classname
@@ -457,8 +471,8 @@ def _doStoreFile(self, name, **databases):
457471
def rollback(self):
458472
''' Reverse all actions from the current transaction.
459473
'''
460-
if hyperdb.DEBUG:
461-
print 'rollback', (self, )
474+
if __debug__:
475+
print >>hyperdb.DEBUG, 'rollback', (self, )
462476
for method, args in self.transactions:
463477
# delete temporary files
464478
if method == self._doStoreFile:
@@ -471,6 +485,9 @@ def rollback(self):
471485

472486
#
473487
#$Log: not supported by cvs2svn $
488+
#Revision 1.33 2002/04/24 10:38:26 rochecompaan
489+
#All database files are now created group readable and writable.
490+
#
474491
#Revision 1.32 2002/04/15 23:25:15 richard
475492
#. node ids are now generated from a lockable store - no more race conditions
476493
#

‎roundup/backends/back_bsddb.py‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
1616
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
1717
#
18-
#$Id: back_bsddb.py,v 1.17 2002-04-03 05:54:31 richard Exp $
18+
#$Id: back_bsddb.py,v 1.18 2002-05-15 06:21:21 richard Exp $
1919
'''
2020
This module defines a backend that saves the hyperdatabase in BSDDB.
2121
'''
@@ -55,18 +55,18 @@ def _opendb(self, name, mode):
5555
'''Low-level database opener that gets around anydbm/dbm
5656
eccentricities.
5757
'''
58-
if hyperdb.DEBUG:
59-
print self, '_opendb', (self, name, mode)
58+
if __debug__:
59+
print >>hyperdb.DEBUG, self, '_opendb', (self, name, mode)
6060
# determine which DB wrote the class file
6161
path = os.path.join(os.getcwd(), self.dir, name)
6262
if not os.path.exists(path):
63-
if hyperdb.DEBUG:
64-
print "_opendb bsddb.open(%r, 'n')"%path
63+
if __debug__:
64+
print >>hyperdb.DEBUG, "_opendb bsddb.open(%r, 'n')"%path
6565
return bsddb.btopen(path, 'n')
6666

6767
# open the database with the correct module
68-
if hyperdb.DEBUG:
69-
print "_opendb bsddb.open(%r, %r)"%(path, mode)
68+
if __debug__:
69+
print >>hyperdb.DEBUG, "_opendb bsddb.open(%r, %r)"%(path, mode)
7070
return bsddb.btopen(path, mode)
7171

7272
#
@@ -102,8 +102,8 @@ def _doSaveJournal(self, classname, nodeid, action, params):
102102
entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
103103
params)
104104

105-
if hyperdb.DEBUG:
106-
print '_doSaveJournal', entry
105+
if __debug__:
106+
print >>hyperdb.DEBUG, '_doSaveJournal', entry
107107

108108
db = bsddb.btopen(os.path.join(self.dir, 'journals.%s'%classname), 'c')
109109

@@ -119,6 +119,14 @@ def _doSaveJournal(self, classname, nodeid, action, params):
119119

120120
#
121121
#$Log: not supported by cvs2svn $
122+
#Revision 1.17 2002/04/03 05:54:31 richard
123+
#Fixed serialisation problem by moving the serialisation step out of the
124+
#hyperdb.Class (get, set) into the hyperdb.Database.
125+
#
126+
#Also fixed htmltemplate after the showid changes I made yesterday.
127+
#
128+
#Unit tests for all of the above written.
129+
#
122130
#Revision 1.16 2002/02/27 03:40:59 richard
123131
#Ran it through pychecker, made fixes
124132
#

‎roundup/cgi_client.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
1616
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
1717
#
18-
# $Id: cgi_client.py,v 1.118 2002-05-12 23:46:33 richard Exp $
18+
# $Id: cgi_client.py,v 1.119 2002-05-15 06:21:21 richard Exp $
1919

2020
__doc__ = """
2121
WWW request handler (also used in the stand-alone server).
@@ -47,6 +47,7 @@ class Client:
4747
'''
4848

4949
def __init__(self, instance, request, env, form=None):
50+
hyperdb.traceMark()
5051
self.instance = instance
5152
self.request = request
5253
self.env = env
@@ -1382,6 +1383,9 @@ def parsePropsFromForm(db, cl, form, nodeid=0):
13821383

13831384
#
13841385
# $Log: not supported by cvs2svn $
1386+
# Revision 1.118 2002/05/12 23:46:33 richard
1387+
# ehem, part 2
1388+
#
13851389
# Revision 1.117 2002/05/12 23:42:29 richard
13861390
# ehem
13871391
#

0 commit comments

Comments
 (0)