Session storage in the hyperdb was horribly, horribly inefficient.

We use a simple anydbm wrapper now - which could be overridden by the
metakit backend or RDB backend if necessary.  Much, much better.

Author: Richard Jones

roundup/backends/back_anydbm.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_anydbm.py,v 1.54 2002-07-26 08:26:59 richard Exp $
+#$Id: back_anydbm.py,v 1.55 2002-07-30 08:22:38 richard Exp $
 '''
 This module defines a backend that saves the hyperdatabase in a database
 chosen by anydbm. It is guaranteed to always be available in python
@@ -26,6 +26,7 @@
 import whichdb, anydbm, os, marshal, re, weakref, string, copy
 from roundup import hyperdb, date, password, roundupdb, security
 from blobfiles import FileStorage
+from sessions import Sessions
 from roundup.indexer import Indexer
 from locking import acquire_lock, release_lock
 from roundup.hyperdb import String, Password, Date, Interval, Link, \
@@ -66,6 +67,7 @@ def __init__(self, config, journaltag=None):
         self.destroyednodes = {}# keep track of the destroyed nodes by class
         self.transactions = []
         self.indexer = Indexer(self.dir)
+        self.sessions = Sessions(self.config)
         self.security = security.Security(self)
         # ensure files are group readable and writable
         os.umask(0002)
@@ -1776,6 +1778,11 @@ def __init__(self, db, classname, **properties):
 
 #
 #$Log: not supported by cvs2svn $
+#Revision 1.54  2002/07/26 08:26:59  richard
+#Very close now. The cgi and mailgw now use the new security API. The two
+#templates have been migrated to that setup. Lots of unit tests. Still some
+#issue in the web form for editing Roles assigned to users.
+#
 #Revision 1.53  2002/07/25 07:14:06  richard
 #Bugger it. Here's the current shape of the new security implementation.
 #Still to do:

roundup/backends/back_metakit.py

@@ -1,5 +1,6 @@
-from roundup import hyperdb, date, password, roundupdb
+from roundup import hyperdb, date, password, roundupdb, security
 import metakit
+from sessions import Sessions
 import re, marshal, os, sys, weakref, time, calendar
 from roundup import indexer 
 
@@ -12,13 +13,15 @@ def __init__(self, config, journaltag=None):
         self.dirty = 0
         self._db = self.__open()
         self.indexer = Indexer(self.config.DATABASE, self._db)
+        self.sessions = Sessions(self.config)
+        self.security = security.Security(self)
+
         os.umask(0002)
     def post_init(self):
         if self.indexer.should_reindex():
             self.reindex()
 
     def reindex(self):
-        print "Reindexing!!!"
         for klass in self.classes.values():
             for nodeid in klass.list():
                 klass.index(nodeid)

roundup/backends/sessions.py

@@ -0,0 +1,101 @@
+#$Id: sessions.py,v 1.1 2002-07-30 08:22:38 richard Exp $
+'''
+This module defines a very basic store that's used by the CGI interface
+to store session information.
+'''
+
+import anydbm, whichdb, os, marshal
+
+class Sessions:
+    ''' Back onto an anydbm store.
+
+        Keys are session id strings, values are marshalled data.
+    '''
+    def __init__(self, config):
+        self.config = config
+        self.dir = config.DATABASE
+        # ensure files are group readable and writable
+        os.umask(0002)
+
+    def clear(self):
+        path = os.path.join(self.dir, 'sessions')
+        if os.path.exists(path):
+            os.remove(path)
+        elif os.path.exists(path+'.db'):    # dbm appends .db
+            os.remove(path+'.db')
+
+    def determine_db_type(self, path):
+        ''' determine which DB wrote the class file
+        '''
+        db_type = ''
+        if os.path.exists(path):
+            db_type = whichdb.whichdb(path)
+            if not db_type:
+                raise hyperdb.DatabaseError, "Couldn't identify database type"
+        elif os.path.exists(path+'.db'):
+            # if the path ends in '.db', it's a dbm database, whether
+            # anydbm says it's dbhash or not!
+            db_type = 'dbm'
+        return db_type
+
+    def get(self, sessionid, value):
+        db = self.opendb('c')
+        try:
+            if db.has_key(sessionid):
+                values = marshal.loads(db[sessionid])
+            else:
+                return None
+            return values.get(value, None)
+        finally:
+            db.close()
+
+    def set(self, sessionid, **newvalues):
+        db = self.opendb('c')
+        try:
+            if db.has_key(sessionid):
+                values = marshal.loads(db[sessionid])
+            else:
+                values = {}
+            values.update(newvalues)
+            db[sessionid] = marshal.dumps(values)
+        finally:
+            db.close()
+
+    def list(self):
+        db = self.opendb('r')
+        try:
+            return db.keys()
+        finally:
+            db.close()
+
+    def destroy(self, sessionid):
+        db = self.opendb('c')
+        try:
+            if db.has_key(sessionid):
+                del db[sessionid]
+        finally:
+            db.close()
+
+    def opendb(self, mode):
+        '''Low-level database opener that gets around anydbm/dbm
+           eccentricities.
+        '''
+        # figure the class db type
+        path = os.path.join(os.getcwd(), self.dir, 'sessions')
+        db_type = self.determine_db_type(path)
+
+        # new database? let anydbm pick the best dbm
+        if not db_type:
+            return anydbm.open(path, 'n')
+
+        # open the database with the correct module
+        dbm = __import__(db_type)
+        return dbm.open(path, mode)
+
+    def commit(self):
+        pass
+
+#
+#$Log: not supported by cvs2svn $
+#
+#

roundup/cgi_client.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: cgi_client.py,v 1.146 2002-07-30 05:27:30 richard Exp $
+# $Id: cgi_client.py,v 1.147 2002-07-30 08:22:38 richard Exp $
 
 __doc__ = """
 WWW request handler (also used in the stand-alone server).
@@ -1197,6 +1197,31 @@ def classes(self, message=None):
         self.write('</table>')
         self.pagefoot()
 
+    def unauthorised(self, message):
+        ''' The user is not authorised to do something. If they're
+            anonymous, throw up a login box. If not, just tell them they
+            can't do whatever it was they were trying to do.
+
+            Bot cases print up the message, which is most likely the
+            argument to the Unauthorised exception.
+        '''
+        self.header(response=403)
+        if self.desired_action is None or self.desired_action == 'login':
+            if not message:
+                message=_("You do not have permission.")
+            action = 'index'
+        else:
+            if not message:
+                message=_("You do not have permission to access"\
+                    " %(action)s.")%{'action': self.desired_action}
+            action = self.desired_action
+        if self.user == 'anonymous':
+            self.login(action=action, message=message)
+        else:
+            self.pagehead(_('Not Authorised'))
+            self.write('<p class="system-msg">%s</p>'%message)
+            self.pagefoot()
+
     def login(self, message=None, newuser_form=None, action='index'):
         '''Display a login page.
         '''
@@ -1331,19 +1356,17 @@ def newuser_action(self, message=None):
 
     def set_cookie(self, user, password):
         # TODO generate a much, much stronger session key ;)
-        session = binascii.b2a_base64(repr(time.time())).strip()
+        self.session = binascii.b2a_base64(repr(time.time())).strip()
 
         # clean up the base64
-        if session[-1] == '=':
-          if session[-2] == '=':
-            session = session[:-2]
-          else:
-            session = session[:-1]
+        if self.session[-1] == '=':
+            if self.session[-2] == '=':
+                self.session = self.session[:-2]
+            else:
+                self.session = self.session[:-1]
 
         # insert the session in the sessiondb
-        sessions = self.db.getclass('__sessions')
-        self.session = sessions.create(sessid=session, user=user,
-            last_use=date.Date())
+        self.db.sessions.set(self.session, user=user, last_use=time.time())
 
         # and commit immediately
         self.db.commit()
@@ -1355,10 +1378,10 @@ def set_cookie(self, user, password):
         path = '/'.join((self.env['SCRIPT_NAME'], self.env['INSTANCE_NAME'],
             ''))
         self.header({'Set-Cookie': 'roundup_user=%s; expires=%s; Path=%s;'%(
-            session, expire, path)})
+            self.session, expire, path)})
 
     def make_user_anonymous(self):
-        ''' Make use anonymous
+        ''' Make us anonymous
 
             This method used to handle non-existence of the 'anonymous'
             user, but that user is mandatory now.
@@ -1367,7 +1390,10 @@ def make_user_anonymous(self):
         self.user = 'anonymous'
 
     def logout(self, message=None):
+        ''' Make us really anonymous - nuke the cookie too
+        '''
         self.make_user_anonymous()
+
         # construct the logout cookie
         now = Cookie._getdate()
         path = '/'.join((self.env['SCRIPT_NAME'], self.env['INSTANCE_NAME'],
@@ -1383,37 +1409,14 @@ def opendb(self, user):
         # open the db
         self.db = self.instance.open(user)
 
-        # make sure we have the session Class
-        try:
-            sessions = self.db.getclass('__sessions')
-        except:
-            # add the sessions Class - use a non-journalling Class
-            # TODO: not happy with how we're getting the Class here :(
-            sessions = self.instance.dbinit.Class(self.db, '__sessions',
-                sessid=hyperdb.String(), user=hyperdb.String(),
-                last_use=hyperdb.Date())
-            sessions.setkey('sessid')
-            # make sure session db isn't journalled
-            sessions.disableJournalling()
-
     def main(self):
         ''' Wrap the request and handle unauthorised requests
         '''
         self.desired_action = None
         try:
             self.main_action()
         except Unauthorised, message:
-            self.header(response=403)
-            if self.desired_action is None or self.desired_action == 'login':
-                if not message:
-                    message=_("You do not have permission.")
-                # go to the index after login
-                self.login(message=message)
-            else:
-                if not message:
-                    message=_("You do not have permission to access"\
-                        " %(action)s.")%{'action': self.desired_action}
-                self.login(action=self.desired_action, message=message)
+            self.unauthorised(message)
 
     def main_action(self):
         '''Wrap the database accesses so we can close the database cleanly
@@ -1422,12 +1425,12 @@ def main_action(self):
         self.opendb('admin')
 
         # make sure we have the session Class
-        sessions = self.db.getclass('__sessions')
+        sessions = self.db.sessions
 
         # age sessions, remove when they haven't been used for a week
         # TODO: this shouldn't be done every access
-        week = date.Interval('7d')
-        now = date.Date()
+        week = 60*60*24*7
+        now = time.time()
         for sessid in sessions.list():
             interval = now - sessions.get(sessid, 'last_use')
             if interval > week:
@@ -1436,22 +1439,21 @@ def main_action(self):
         # look up the user session cookie
         cookie = Cookie.Cookie(self.env.get('HTTP_COOKIE', ''))
         user = 'anonymous'
+
         if (cookie.has_key('roundup_user') and
                 cookie['roundup_user'].value != 'deleted'):
 
             # get the session key from the cookie
-            session = cookie['roundup_user'].value
+            self.session = cookie['roundup_user'].value
 
             # get the user from the session
             try:
-                self.session = sessions.lookup(session)
+                # update the lifetime datestamp
+                sessions.set(self.session, last_use=time.time())
+                sessions.commit()
+                user = sessions.get(self.session, 'user')
             except KeyError:
                 user = 'anonymous'
-            else:
-                # update the lifetime datestamp
-                sessions.set(self.session, last_use=date.Date())
-                self.db.commit()
-                user = sessions.get(sessid, 'user')
 
         # sanity check on the user still being valid
         try:
@@ -1689,6 +1691,9 @@ def parsePropsFromForm(db, cl, form, nodeid=0, num_re=re.compile('^\d+$')):
 
 #
 # $Log: not supported by cvs2svn $
+# Revision 1.146  2002/07/30 05:27:30  richard
+# nicer error messages, and a bugfix
+#
 # Revision 1.145  2002/07/26 08:26:59  richard
 # Very close now. The cgi and mailgw now use the new security API. The two
 # templates have been migrated to that setup. Lots of unit tests. Still some