More indexer work:

- all String properties may now be indexed too. Currently there's a bit of
  "issue" specific code in the actual searching which needs to be
  addressed. In a nutshell:
  + pass 'indexme="yes"' as a String() property initialisation arg, eg:
        file = FileClass(db, "file", name=String(), type=String(),
            comment=String(indexme="yes"))
  + the comment will then be indexed and be searchable, with the results
    related back to the issue that the file is linked to
- as a result of this work, the FileClass has a default MIME type that may
  be overridden in a subclass, or by the use of a "type" property as is
  done in the default templates.
- the regeneration of the indexes (if necessary) is done once the schema is
  set up in the dbinit.

Author: Richard Jones

CHANGES.txt

@@ -12,6 +12,19 @@ Fixed:
    - only index text/plain and rfc822/message (ideas for other text formats to
      index are welcome)
    - added simple unit test for indexer. Needs more tests for regression.
+   - all String properties may now be indexed too. Currently there's a bit of
+     "issue" specific code in the actual searching which needs to be
+     addressed. In a nutshell:
+     + pass 'indexme="yes"' as a String() property initialisation arg, eg:
+           file = FileClass(db, "file", name=String(), type=String(),
+               comment=String(indexme="yes"))
+     + the comment will then be indexed and be searchable, with the results
+       related back to the issue that the file is linked to
+   - as a result of this work, the FileClass has a default MIME type that may
+     be overridden in a subclass, or by the use of a "type" property as is
+     done in the default templates.
+   - the regeneration of the indexes (if necessary) is done once the schema is
+     set up in the dbinit.
  . made the unit tests run again - they were quite b0rken
 
 

doc/upgrading.txt

@@ -7,6 +7,17 @@ accordingly.
 
 .. contents::
 
+Migrating from 0.4.x to 0.5.0
+=============================
+
+0.5.0 Configuration
+-------------------
+
+TODO: mention stuff about indexing
+TODO: mention that the dbinit needs the db.post_init() method call for
+reindexing
+
+
 Migrating from 0.4.1 to 0.4.2
 =============================
 

roundup/backends/back_anydbm.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_anydbm.py,v 1.38 2002-07-08 06:58:15 richard Exp $
+#$Id: back_anydbm.py,v 1.39 2002-07-09 03:02:52 richard Exp $
 '''
 This module defines a backend that saves the hyperdatabase in a database
 chosen by anydbm. It is guaranteed to always be available in python
@@ -66,6 +66,16 @@ def __init__(self, config, journaltag=None):
         # ensure files are group readable and writable
         os.umask(0002)
 
+    def post_init(self):
+        """Called once the schema initialisation has finished."""
+        # reindex the db if necessary
+        if not self.indexer.should_reindex():
+            return
+        for klass in self.classes.values():
+            for nodeid in klass.list():
+                klass.index(nodeid)
+        self.indexer.save_index()
+
     def __repr__(self):
         return '<back_anydbm instance at %x>'%id(self) 
 
@@ -409,15 +419,24 @@ def commit(self):
         self.databases = {}
 
         # now, do all the transactions
+        reindex = {}
         for method, args in self.transactions:
-            method(*args)
+            reindex[method(*args)] = 1
 
         # now close all the database files
         for db in self.databases.values():
             db.close()
         del self.databases
         # TODO: unlock the DB
 
+        # reindex the nodes that request it
+        for classname, nodeid in filter(None, reindex.keys()):
+            print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid)
+            self.getclass(classname).index(nodeid)
+
+        # save the indexer state
+        self.indexer.save_index()
+
         # all transactions committed, back to normal
         self.cache = {}
         self.dirtynodes = {}
@@ -439,6 +458,9 @@ def _doSaveNode(self, classname, nodeid, node):
         # now save the marshalled data
         db[nodeid] = marshal.dumps(self.serialise(classname, node))
 
+        # return the classname, nodeid so we reindex this content
+        return (classname, nodeid)
+
     def _doSaveJournal(self, classname, nodeid, action, params):
         # serialise first
         if action in ('set', 'create'):
@@ -477,15 +499,23 @@ def rollback(self):
         for method, args in self.transactions:
             # delete temporary files
             if method == self._doStoreFile:
-                if os.path.exists(args[0]+".tmp"):
-                    os.remove(args[0]+".tmp")
+                self._rollbackStoreFile(*args)
         self.cache = {}
         self.dirtynodes = {}
         self.newnodes = {}
         self.transactions = []
 
 #
 #$Log: not supported by cvs2svn $
+#Revision 1.38  2002/07/08 06:58:15  richard
+#cleaned up the indexer code:
+# - it splits more words out (much simpler, faster splitter)
+# - removed code we'll never use (roundup.roundup_indexer has the full
+#   implementation, and replaces roundup.indexer)
+# - only index text/plain and rfc822/message (ideas for other text formats to
+#   index are welcome)
+# - added simple unit test for indexer. Needs more tests for regression.
+#
 #Revision 1.37  2002/06/20 23:52:35  richard
 #More informative error message
 #

roundup/backends/blobfiles.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: blobfiles.py,v 1.5 2002-07-08 06:58:15 richard Exp $
+#$Id: blobfiles.py,v 1.6 2002-07-09 03:02:52 richard Exp $
 '''
 This module exports file storage for roundup backends.
 Files are stored into a directory hierarchy.
@@ -74,12 +74,19 @@ def storefile(self, classname, nodeid, property, content):
            None, in which case the filename does not indicate which property
            is being saved.
         '''
+        # determine the name of the file to write to
         name = self.filename(classname, nodeid, property)
+
+        # make sure the file storage dir exists
         if not os.path.exists(os.path.dirname(name)):
             os.makedirs(os.path.dirname(name))
+
+        # open the temp file for writing
         open(name + '.tmp', 'wb').write(content)
-        self.transactions.append((self._doStoreFile, (name, )))
 
+        # save off the commit action
+        self.transactions.append((self._doStoreFile, (classname, nodeid,
+            property)))
 
     def getfile(self, classname, nodeid, property):
         '''Get the content of the file in the database.
@@ -88,6 +95,7 @@ def getfile(self, classname, nodeid, property):
         try:
             return open(filename, 'rb').read()
         except:
+            # now try the temp pre-commit filename
             try:
                 return open(filename+'.tmp', 'rb').read()
             except:
@@ -101,15 +109,36 @@ def numfiles(self):
         files_dir = os.path.join(self.dir, 'files')
         return files_in_dir(files_dir)
 
-    def _doStoreFile(self, name, **databases):
+    def _doStoreFile(self, classname, nodeid, property, **databases):
         '''Store the file as part of a transaction commit.
         '''
+        # determine the name of the file to write to
+        name = self.filename(classname, nodeid, property)
+
         # the file is currently ".tmp" - move it to its real name to commit
         os.rename(name+".tmp", name)
-        self.indexer.add_file(name)
-        self.indexer.save_index()
+
+        # return the classname, nodeid so we reindex this content
+        return (classname, nodeid)
+
+    def _rollbackStoreFile(self, classname, nodeid, property, **databases):
+        '''Remove the temp file as a part of a rollback
+        '''
+        # determine the name of the file to delete
+        name = self.filename(classname, nodeid, property)
+        if os.path.exists(name+".tmp"):
+            os.remove(name+".tmp")
 
 # $Log: not supported by cvs2svn $
+# Revision 1.5  2002/07/08 06:58:15  richard
+# cleaned up the indexer code:
+#  - it splits more words out (much simpler, faster splitter)
+#  - removed code we'll never use (roundup.roundup_indexer has the full
+#    implementation, and replaces roundup.indexer)
+#  - only index text/plain and rfc822/message (ideas for other text formats to
+#    index are welcome)
+#  - added simple unit test for indexer. Needs more tests for regression.
+#
 # Revision 1.4  2002/06/19 03:07:19  richard
 # Moved the file storage commit into blobfiles where it belongs.
 #

roundup/hyperdb.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: hyperdb.py,v 1.70 2002-06-27 12:06:20 gmcm Exp $
+# $Id: hyperdb.py,v 1.71 2002-07-09 03:02:52 richard Exp $
 
 __doc__ = """
 Hyperdatabase implementation, especially field types.
@@ -56,6 +56,8 @@ def traceMark():
 #
 class String:
     """An object designating a String property."""
+    def __init__(self, indexme='no'):
+        self.indexme = indexme == 'yes'
     def __repr__(self):
         ' more useful for dumps '
         return '<%s>'%self.__class__
@@ -157,6 +159,10 @@ def __init__(self, config, journaltag=None):
         """
         raise NotImplementedError
 
+    def post_init(self):
+        """Called once the schema initialisation has finished."""
+        raise NotImplementedError
+
     def __getattr__(self, classname):
         """A convenient way of calling self.getclass(classname)."""
         raise NotImplementedError
@@ -1111,6 +1117,16 @@ def addprop(self, **properties):
                 raise ValueError, key
         self.properties.update(properties)
 
+    def index(self, nodeid):
+        '''Add (or refresh) the node to search indexes
+        '''
+        # find all the String properties that have indexme
+        for prop, propclass in self.getprops().items():
+            if isinstance(propclass, String) and propclass.indexme:
+                # and index them under (classname, nodeid, property)
+                self.db.indexer.add_text((self.classname, nodeid, prop),
+                    str(self.get(nodeid, prop)))
+
 # XXX not in spec
 class Node:
     ''' A convenience wrapper for the given node
@@ -1169,6 +1185,9 @@ def Choice(name, db, *options):
 
 #
 # $Log: not supported by cvs2svn $
+# Revision 1.70  2002/06/27 12:06:20  gmcm
+# Improve an error message.
+#
 # Revision 1.69  2002/06/17 23:15:29  richard
 # Can debug to stdout now
 #