Add tests to BasicDatabase and merge implementations

test/session_common.py:

  add new tests:

  * test set with bad __timestamps
  * test get on missing item with no default
  * test clear() method
  * test new lifetime() method

  everything below here was needed to get the tests to work across all
  db's.

roundup/backends/sessions_dbm.py:

  make set() validate __timestamp as float. If invalid and item is new,
  set it to time.time(). If invalid and item exists keep original
  timestamp.

  add lifetime(key_lifetime) method. Given key_lifetime in seconds,
  generate a __timestamp that will be deleted after that many seconds.

roundup/backends/sessions_rdbms.py:

  make set() behave the same as session_dbm.

  add lifetime method as above.

roundup/backends/sessions_sqlite.py:

  import session_rdbms::BasicDatabase and override __init__.
  rather than cloning all the code. Kept a few logging and sql methods.

roundup/test/memorydb.py:

  make get() on a missing key return KeyError
  make set() conform to dbm/rdbms implementations.

Author: rouilj

roundup/backends/sessions_dbm.py

@@ -88,11 +88,26 @@ def getall(self, infoid):
 
     def set(self, infoid, **newvalues):
         db = self.opendb('c')
+        timestamp=None
         try:
             if infoid in db:
                 values = marshal.loads(db[infoid])
+                try:
+                    timestamp = values['__timestamp']
+                except KeyError:
+                    pass  # stay at None
             else:
-                values = {'__timestamp': time.time()}
+                values = {}
+
+            if '__timestamp' in newvalues:
+                try:
+                    float(newvalues['__timestamp'])
+                except ValueError:
+                    # keep original timestamp if present
+                    newvalues['__timestamp'] = timestamp or time.time()
+            else:
+                newvalues['__timestamp'] = time.time()
+
             values.update(newvalues)
             db[infoid] = marshal.dumps(values)
         finally:
@@ -157,6 +172,14 @@ def opendb(self, mode):
     def commit(self):
         pass
 
+    def lifetime(self, key_lifetime=0):
+        """Return the proper timestamp for a key with key_lifetime specified
+           in seconds. Default lifetime is 0.
+        """
+        now = time.time()
+        week = 60*60*24*7
+        return now - week + key_lifetime
+
     def close(self):
         pass
 

roundup/backends/sessions_rdbms.py

@@ -60,37 +60,34 @@ def set(self, infoid, **newvalues):
         c = self.cursor
         n = self.name
         a = self.db.arg
-        c.execute('select %s_value, %s_time from %ss where %s_key=%s'% \
-                  (n, n, n, n, a),
+        c.execute('select %s_value from %ss where %s_key=%s'% \
+                  (n, n, n, a),
             (infoid,))
         res = c.fetchone()
+
+        timestamp=time.time()
         if res:
             values = eval(res[0])
-            timestamp = res[1]
         else:
             values = {}
+
+        if '__timestamp' in newvalues:
+            try:
+                # __timestamp must be representable as a float. Check it.
+                timestamp = float(newvalues['__timestamp'])
+            except ValueError:
+                if res:
+                    # keep the original timestamp
+                    del(newvalues['__timestamp'])
+                else:
+                    # here timestamp is the new timestamp
+                    newvalues['__timestamp'] = timestamp
         values.update(newvalues)
         if res:
-            if '__timestamp' in newvalues:
-                try:
-                    # __timestamp must be representable as a float. Check it.
-                    timestamp = float(newvalues['__timestamp'])
-                except ValueError:
-                    pass
-
             sql = ('update %ss set %s_value=%s, %s_time=%s '
                        'where %s_key=%s'%(n, n, a, n, a, n, a))
             args = (repr(values), timestamp, infoid)
         else:
-            if '__timestamp' in newvalues:
-                try:
-                    # __timestamp must be represntable as a float. Check it.
-                    timestamp = float(newvalues['__timestamp'])
-                except ValueError:
-                    timestamp = time.time()
-            else:
-                timestamp = time.time()
-
             sql = 'insert into %ss (%s_key, %s_time, %s_value) '\
                 'values (%s, %s, %s)'%(n, n, n, n, a, a, a)
             args = (infoid, timestamp, repr(values))
@@ -128,6 +125,14 @@ def commit(self):
         self.conn.commit()
         self.cursor = self.conn.cursor()
 
+    def lifetime(self, item_lifetime=0):
+        """Return the proper timestamp for a key with key_lifetime specified
+           in seconds. Default lifetime is 0.
+        """
+        now = time.time()
+        week = 60*60*24*7
+        return now - week + item_lifetime
+
     def close(self):
         self.conn.close()
 

roundup/backends/sessions_sqlite.py

@@ -14,8 +14,9 @@
 import os, time, logging
 
 from roundup.anypy.html import html_escape as escape
+import roundup.backends.sessions_rdbms as rdbms_session
 
-class BasicDatabase:
+class BasicDatabase(rdbms_session.BasicDatabase):
     ''' Provide a nice encapsulation of an RDBMS table.
 
         Keys are id strings, values are automatically marshalled data.
@@ -68,118 +69,6 @@ def sql(self, sql, args=None, cursor=None):
         else:
             cursor.execute(sql)
 
-    def clear(self):
-        self.cursor.execute('delete from %ss'%self.name)
-
-    def exists(self, infoid):
-        n = self.name
-        self.cursor.execute('select count(*) from %ss where %s_key=%s'%(n,
-            n, self.db.arg), (infoid,))
-        return int(self.cursor.fetchone()[0])
-
-    _marker = []
-    def get(self, infoid, value, default=_marker):
-        n = self.name
-        self.cursor.execute('select %s_value from %ss where %s_key=%s'%(n,
-            n, n, self.db.arg), (infoid,))
-        res = self.cursor.fetchone()
-        if not res:
-            if default != self._marker:
-                return default
-            raise KeyError('No such %s "%s"'%(self.name, escape(infoid)))
-        values = eval(res[0])
-        return values.get(value, None)
-
-    def getall(self, infoid):
-        n = self.name
-        self.cursor.execute('select %s_value from %ss where %s_key=%s'%(n,
-            n, n, self.db.arg), (infoid,))
-        res = self.cursor.fetchone()
-        if not res:
-            raise KeyError('No such %s "%s"'%(self.name, escape (infoid)))
-        return eval(res[0])
-
-    def set(self, infoid, **newvalues):
-        """ Store all newvalues under key infoid with a timestamp in database.
-
-            If newvalues['__timestamp'] exists and is representable as a floating point number
-            (i.e. could be generated by time.time()), that value is used for the <name>_time
-            column in the database.
-        """
-        c = self.cursor
-        n = self.name
-        a = self.db.arg
-        c.execute('select %s_value from %ss where %s_key=%s'%(n, n, n, a),
-            (infoid,))
-        res = c.fetchone()
-        if res:
-            values = eval(res[0])
-        else:
-            values = {}
-        values.update(newvalues)
-
-        if res:
-            sql = 'update %ss set %s_value=%s where %s_key=%s'%(n, n,
-                a, n, a)
-            args = (repr(values), infoid)
-        else:
-            if '__timestamp' in newvalues:
-                try:
-                    # __timestamp must be represntable as a float. Check it.
-                    timestamp = float(newvalues['__timestamp'])
-                except ValueError:
-                    timestamp = time.time()
-            else:
-                timestamp = time.time()
-
-            sql = 'insert into %ss (%s_key, %s_time, %s_value) '\
-                'values (%s, %s, %s)'%(n, n, n, n, a, a, a)
-            args = (infoid, timestamp, repr(values))
-        c.execute(sql, args)
-
-    def list(self):
-        c = self.cursor
-        n = self.name
-        c.execute('select %s_key from %ss'%(n, n))
-        return [res[0] for res in c.fetchall()]
-
-    def destroy(self, infoid):
-        self.cursor.execute('delete from %ss where %s_key=%s'%(self.name,
-            self.name, self.db.arg), (infoid,))
-
-    def updateTimestamp(self, infoid):
-        """ don't update every hit - once a minute should be OK """
-        now = time.time()
-        self.cursor.execute('''update %ss set %s_time=%s where %s_key=%s
-            and %s_time < %s'''%(self.name, self.name, self.db.arg,
-            self.name, self.db.arg, self.name, self.db.arg),
-            (now, infoid, now-60))
-
-    def clean(self):
-        ''' Remove session records that haven't been used for a week. '''
-        now = time.time()
-        week = 60*60*24*7
-        old = now - week
-        self.cursor.execute('delete from %ss where %s_time < %s'%(self.name,
-            self.name, self.db.arg), (old, ))
-
-    def lifetime(self, key_lifetime=None):
-        """Return the proper timestamp for a key with key_lifetime specified
-           in seconds.
-        """
-        now = time.time()
-        week = 60*60*24*7
-        return now - week + lifetime
-
-    def commit(self):
-        logger = logging.getLogger('roundup.hyperdb.backend')
-        logger.info('commit %s' % self.name)
-        self.conn.commit()
-        self.cursor = self.conn.cursor()
-
-    def close(self):
-        self.conn.close()
-
 class Sessions(BasicDatabase):
     name = 'session'
 

roundup/test/memorydb.py

@@ -3,6 +3,7 @@
 
 import shutil
 import os
+import time
 
 from roundup import date
 from roundup import hyperdb
@@ -163,12 +164,22 @@ def __contains__(self, key):
     def exists(self, infoid):
         return infoid in self
     def get(self, infoid, value, default=None):
+        if infoid not in self:
+            raise KeyError
         return self[infoid].get(value, default)
     def getall(self, infoid):
         if infoid not in self:
             raise KeyError(infoid)
         return self[infoid]
     def set(self, infoid, **newvalues):
+        if '__timestamp' in newvalues:
+            try:
+                float(newvalues['__timestamp'])
+            except ValueError:
+                if infoid in self:
+                    del(newvalues['__timestamp'])
+                else:
+                    newvalues['__timestamp'] = time.time()
         self[infoid].update(newvalues)
     def list(self):
         return list(self.keys())

test/session_common.py

@@ -2,7 +2,28 @@
 
 from .db_test_base import config
 
+"""
+here are three different impementations for these. I am trying to fix
+them so they all act the same.
 
+set with invalid timestamp:
+
+   session_dbm/memorydb - sets to invalid timestamp if new or existing item.
+   session_rdbms - sets to time.time if new item, keeps original
+                   if item exists. (note that the timestamp is
+                   a separate column, the timestamp embedded in the
+                   value object in the db has the bad __timestamp.
+   reconciled: set to time.time for new item, keeps original time
+               of existing item.
+
+Also updateTimestamp does not update the marshalled values idea of
+   __timestamp. So get(item, '__timestamp') will not work as expected
+   for rdbms backends, need a sql query to get the timestamp column.
+
+FIXME need to add getTimestamp method to sessions_rdbms.py and
+sessions_dbm.py.
+
+"""
 class SessionTest(object):
     def setUp(self):
         # remove previous test, ignore errors
@@ -33,6 +54,11 @@ def testList(self):
         self.assertEqual(self.sessions.list().sort(),
                 [self.s2b('random_key'), self.s2b('random_key2')].sort())
 
+    def testGetMissingKey(self):
+        self.sessions.set('random_key', text='hello, world!', otherval='bar')
+        with self.assertRaises(KeyError) as e:
+            self.sessions.get('badc_key', 'text')
+
     def testGetAll(self):
         self.sessions.set('random_key', text='hello, world!', otherval='bar')
         self.assertEqual(self.sessions.getall('random_key'),
@@ -45,6 +71,16 @@ def testDestroy(self):
         self.sessions.destroy('random_key')
         self.assertRaises(KeyError, self.sessions.getall, 'random_key')
 
+    def testClear(self):
+        self.sessions.set('random_key', text='hello, world!')
+        self.sessions.set('random_key2', text='hello, world!')
+        self.sessions.set('random_key3', text='hello, world!')
+        self.assertEqual(self.sessions.getall('random_key3'),
+            {'text': 'hello, world!'})
+        self.assertEqual(len(self.sessions.list()), 3)
+        self.sessions.clear()
+        self.assertEqual(len(self.sessions.list()), 0)
+
     def testSetSession(self):
         self.sessions.set('random_key', text='hello, world!', otherval='bar')
         self.assertEqual(self.sessions.get('random_key', 'text'),
@@ -59,6 +95,32 @@ def testUpdateSession(self):
         self.sessions.set('random_key', text='nope')
         self.assertEqual(self.sessions.get('random_key', 'text'), 'nope')
 
+    def testBadTimestamp(self):
+        self.sessions.set('random_key',
+                          text='hello, world!',
+                          __timestamp='not a timestamp')
+        ts = self.sessions.get('random_key', '__timestamp')
+        self.assertNotEqual(ts, 'not a timestamp')
+        # use {1,7} because db's don't pad the fraction to 7 digits.
+        ts_re=r'^[0-9]{10,16}\.[0-9]{1,7}$'
+        try:
+            self.assertRegex(str(ts), ts_re)
+        except AttributeError:   # 2.7 version
+            import warnings
+            with warnings.catch_warnings():
+                warnings.filterwarnings("ignore",category=DeprecationWarning)
+                self.assertRegexpMatches(str(ts), ts_re)
+
+        # now update with a bad timestamp, original timestamp should
+        # be kept.
+        self.sessions.set('random_key',
+                          text='hello, world2!',
+                          __timestamp='not a timestamp')
+        item = self.sessions.get('random_key', "text")
+        item_ts = self.sessions.get('random_key', "__timestamp")
+        self.assertEqual(item, 'hello, world2!')
+        self.assertAlmostEqual(ts, item_ts, 2)
+
     # overridden in dbm and memory backends
     def testUpdateTimestamp(self):
         def get_ts_via_sql(self):
@@ -88,3 +150,9 @@ def get_ts_via_sql(self):
 
         # use 61 to allow a fudge factor
         self.assertGreater(get_ts_via_sql(self)[0] - timestamp, 61)
+
+    def testLifetime(self):
+        ts = self.sessions.lifetime(300)
+        week_ago =  time.time() - 60*60*24*7
+        self.assertGreater(week_ago + 302, ts)
+        self.assertLess(week_ago + 298, ts)