Set all sqlite db's to WAL mode on creation

Complete work done on 6917:fba76e0bba98.

WAL journaling mode now enabled on primary db as well as session db's
when using sqlite for session databases.

Made change to WAL mode more robust for testing by bracketing
`pragma journal_mode=wal` with commit() calls. Normally the conversion
would occur when a new session was opened, but this commits the changes
explicitly to make all the tests stable.

Also added doc on this.

Author: rouilj

CHANGES.txt

@@ -59,6 +59,9 @@ Features:
 - sqlite native-fts backend now uses the stopwords list in config.ini
   to filter words from queries. (Stopwords are still indexed so that
   phrase/proximity searches still work.) (John Rouillard)
+- sqlite databases use WAL mode when *created* to improve read
+  concurrency. Existing sqlite database still use rollback journal
+  mode. See upgrading.txt for details. (John Rouillard)
 
 2022-07-13 2.2.0
 

doc/upgrading.txt

@@ -124,6 +124,51 @@ Session Databases`_ in the `administration guide`_
 .. _Using Redis for Session Databases:
     admin_guide.html#using-redis-for-session-databases
 
+New SQLite databases created with WAL mode journaling (optional)
+----------------------------------------------------------------
+
+By default, SQLite databases use a rollback journal when
+writing an update. The rollback journal stores a copy of the
+data from before the update. One downside of this is that
+all reads have to be suspended while a write is
+occurring. SQLite has an alternate way of insuring ACID
+compliance by using a WAL (write ahead log) journal.
+
+Version 2.3.0 of Roundup, creates new SQLite databases using
+WAL journaling. With WAL, a writer does not block readers
+and readers do not block writing an update. This keeps
+Roundup accessible even under a heavy write load (e.g. when
+bulk loading data or automated updates via REST).
+
+If you want to convert your existing SQLite db to WAL mode:
+
+   1. check the current journal mode on your database
+      using::
+
+        sqlite3 <tracker_home>/db/db "pragma journal_mode;"
+
+   2. If it returns ``delete``, change it to WAL mode using::
+
+        sqlite3 <tracker_home>/db/db "pragma journal_mode=WAL;"
+ 
+   3. verify by running the command in step 1 again and you
+      should get ``wal``.
+
+If you are using SQLite for session and otk databases,
+perform the same steps replacing ``db`` with ``db-session``
+and ``db-otk``.
+
+If you find WAL mode is not working for you, you can set the
+journal method to a rollback journal (``delete`` mode) by
+using step 2 and replacing ``wal`` with ``delete``. (Note:
+SQLite supports other journaling modes, but only ``wal`` and
+``delete`` persist. Roundup doesn't set a journaling mode
+when it opens the database, so options such as ``truncate``
+are not used.)
+
+For details on WAL mode see `<https://www.sqlite.org/wal.html>`_
+and `<https://www.sqlite.org/pragma.html#pragma_journal_mode>`_.
+
 .. index:: Upgrading; 2.1.0 to 2.2.0
 
 Migrating from 2.1.0 to 2.2.0

roundup/backends/back_sqlite.py

@@ -204,7 +204,9 @@ def open_connection(self):
             self.create_version_2_tables()
             self._add_fts5_table()
             # Set journal mode to WAL.
-            self.conn.execute('pragma journal_mode=wal')
+            self.sql_commit()  # close out rollback journal/transaction
+            self.sql('pragma journal_mode=wal')  # set wal
+            self.sql_commit()  # close out rollback and commit wal change
 
     def create_version_2_tables(self):
         self.sql('create table otks (otk_key varchar, '

roundup/backends/sessions_sqlite.py

@@ -36,7 +36,10 @@ def __init__(self, db):
             %(name)s_value TEXT, %(name)s_time REAL)''' % {"name": self.name})
             self.sql('CREATE INDEX %(name)s_key_idx ON '
                      '%(name)ss(%(name)s_key)' % {"name": self.name})
-            self.commit()
+            # Set journal mode to WAL.
+            self.commit()  # close out rollback journal/transaction
+            self.sql('pragma journal_mode=wal')  # set wal
+            self.commit()  # close out rollback and commit wal change
 
     def sql(self, sql, args=None, cursor=None):
         """ Execute the sql with the optional args.

test/test_sqlite.py

@@ -33,6 +33,18 @@ class sqliteOpener:
     def nuke_database(self):
         shutil.rmtree(config.DATABASE)
 
+    def testWalMode(self):
+        """verify that all sqlite db's are in WAL mode
+           and not journal mode
+        """
+        if not hasattr(self, 'db'):
+            self.skipTest("test has no database open")
+
+        for db in [self.db]:
+            print("testing db", str(db))
+            db.sql('pragma journal_mode;')
+            self.assertEqual(db.cursor.fetchone()['journal_mode'], 'wal')
+
 
 class sqliteDBTest(sqliteOpener, DBTest, unittest.TestCase):
 
@@ -198,6 +210,15 @@ def testDbType(self):
         self.assertIn("roundlite", repr(self.db))
         self.assertIn("roundup.backends.sessions_sqlite.Sessions", repr(self.db.Session))
 
+    def testWalMode(self):
+        """verify that all sqlite db's are in WAL mode
+           and not Rollback mode
+        """
+        for db in [self.db, self.db.Session, self.db.Otk]:
+            print("testing db", str(db))
+            db.sql('pragma journal_mode;')
+            self.assertEqual(db.cursor.fetchone()['journal_mode'], 'wal')
+
 class anydbmSessionTest(sqliteOpener, SessionTest, unittest.TestCase):
     s2b = lambda x,y : y
 
@@ -227,6 +248,7 @@ def get_ts(self):
     def testDbType(self):
         self.assertIn("roundlite", repr(self.db))
         self.assertIn("roundup.backends.sessions_dbm.Sessions", repr(self.db.Session))
-    
+
+  
 class sqliteRestTest (RestTestCase, unittest.TestCase):
     backend = 'sqlite'