A few big changes in this commit:

1. The current indexer has been moved to backends/indexer_dbm in
   anticipation of my writing an indexer_rdbms,
2. Changed indexer invocation during create / set to follow the pattern
   set by the metakit backend, which was much cleaner, and
3. The "content" property of FileClass is now mutable in all but the
   metakit backend.

Metakit needs to be changed to support the editing of "content". Hey, and
I learnt today that the metakit backend implements its own indexer. How
about that... :)

Author: Richard Jones

CHANGES.txt

@@ -55,6 +55,7 @@ Fixed:
 - the mail gateway now searches recursively for the text/plain and the
   attachments of a message (sf bug 841241).
 - fixed display of feedback messages in some situations (sf bug 739545)
+- fixed ability to edit "content" property (sf bug 914062)
 
 Cleanup:
 - replace curuserid attribute on Database with the extended getuid() method.

roundup/backends/back_anydbm.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: back_anydbm.py,v 1.138 2004-03-18 01:58:45 richard Exp $
+#$Id: back_anydbm.py,v 1.139 2004-03-19 04:47:59 richard Exp $
 '''This module defines a backend that saves the hyperdatabase in a
 database chosen by anydbm. It is guaranteed to always be available in python
 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
@@ -37,7 +37,7 @@
 from roundup import hyperdb, date, password, roundupdb, security
 from blobfiles import FileStorage
 from sessions_dbm import Sessions, OneTimeKeys
-from roundup.indexer import Indexer
+from indexer_dbm import Indexer
 from roundup.backends import locking
 from roundup.hyperdb import String, Password, Date, Interval, Link, \
     Multilink, DatabaseError, Boolean, Number, Node
@@ -882,6 +882,7 @@ def create_inner(self, **propvalues):
             elif isinstance(prop, String):
                 if type(value) != type('') and type(value) != type(u''):
                     raise TypeError, 'new property "%s" not a string'%key
+                self.db.indexer.add_text((self.classname, newid, key), value)
 
             elif isinstance(prop, Password):
                 if not isinstance(value, password.Password):
@@ -1143,6 +1144,15 @@ class or a KeyError is raised.
         These operations trigger detectors and can be vetoed.  Attempts
         to modify the "creation" or "activity" properties cause a KeyError.
         '''
+        self.fireAuditors('set', nodeid, propvalues)
+        oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
+        propvalues = self.set_inner(nodeid, **propvalues)
+        self.fireReactors('set', nodeid, oldvalues)
+        return propvalues        
+
+    def set_inner(self, nodeid, **propvalues):
+        ''' Called by set, in-between the audit and react calls.
+        '''
         if not propvalues:
             return propvalues
 
@@ -1155,11 +1165,6 @@ class or a KeyError is raised.
         if self.db.journaltag is None:
             raise DatabaseError, 'Database open read-only'
 
-        self.fireAuditors('set', nodeid, propvalues)
-        # Take a copy of the node dict so that the subsequent set
-        # operation doesn't modify the oldvalues structure.
-        oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid))
-
         node = self.db.getnode(self.classname, nodeid)
         if node.has_key(self.db.RETIRED_FLAG):
             raise IndexError
@@ -1290,6 +1295,8 @@ class or a KeyError is raised.
             elif isinstance(prop, String):
                 if value is not None and type(value) != type('') and type(value) != type(u''):
                     raise TypeError, 'new property "%s" not a string'%propname
+                self.db.indexer.add_text((self.classname, nodeid, propname),
+                    value)
 
             elif isinstance(prop, Password):
                 if not isinstance(value, password.Password):
@@ -1331,9 +1338,7 @@ class or a KeyError is raised.
         if self.do_journal:
             self.db.addjournal(self.classname, nodeid, 'set', journalvalues)
 
-        self.fireReactors('set', nodeid, oldvalues)
-
-        return propvalues        
+        return propvalues
 
     def retire(self, nodeid):
         '''Retire a node.
@@ -1946,20 +1951,18 @@ def addprop(self, **properties):
         self.properties.update(properties)
 
     def index(self, nodeid):
-        '''Add (or refresh) the node to search indexes
-        '''
+        ''' Add (or refresh) the node to search indexes '''
         # find all the String properties that have indexme
         for prop, propclass in self.getprops().items():
-            if isinstance(propclass, String) and propclass.indexme:
+            if isinstance(propclass, hyperdb.String) and propclass.indexme:
+                # index them under (classname, nodeid, property)
                 try:
                     value = str(self.get(nodeid, prop))
                 except IndexError:
-                    # node no longer exists - entry should be removed
-                    self.db.indexer.purge_entry((self.classname, nodeid, prop))
-                else:
-                    # and index them under (classname, nodeid, property)
-                    self.db.indexer.add_text((self.classname, nodeid, prop),
-                        value)
+                    # node has been destroyed
+                    continue
+                self.db.indexer.add_text((self.classname, nodeid, prop), value)
+
 
     #
     # Detector interface
@@ -2012,8 +2015,15 @@ def create(self, **propvalues):
         content = propvalues['content']
         del propvalues['content']
 
+        # make sure we have a MIME type
+        mime_type = propvalues.get('type', self.default_mime_type)
+
         # do the database create
-        newid = Class.create_inner(self, **propvalues)
+        newid = self.create_inner(**propvalues)
+
+        # and index!
+        self.db.indexer.add_text((self.classname, newid, 'content'), content,
+            mime_type)
 
         # fire reactors
         self.fireReactors('create', newid, None)
@@ -2059,6 +2069,35 @@ def get(self, nodeid, propname, default=_marker, cache=1):
         else:
             return Class.get(self, nodeid, propname)
 
+    def set(self, itemid, **propvalues):
+        ''' Snarf the "content" propvalue and update it in a file
+        '''
+        self.fireAuditors('set', itemid, propvalues)
+        oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid))
+
+        # now remove the content property so it's not stored in the db
+        content = None
+        if propvalues.has_key('content'):
+            content = propvalues['content']
+            del propvalues['content']
+
+        # do the database create
+        propvalues = self.set_inner(itemid, **propvalues)
+
+        # do content?
+        if content:
+            # store and index
+            self.db.storefile(self.classname, itemid, None, content)
+            mime_type = propvalues.get('type', self.get(itemid, 'type'))
+            if not mime_type:
+                mime_type = self.default_mime_type
+            self.db.indexer.add_text((self.classname, itemid, 'content'),
+                content, mime_type)
+
+        # fire reactors
+        self.fireReactors('set', itemid, oldvalues)
+        return propvalues
+
     def getprops(self, protected=1):
         ''' In addition to the actual properties on the node, these methods
             provide the "content" property. If the "protected" flag is true,
@@ -2069,27 +2108,6 @@ def getprops(self, protected=1):
         d['content'] = hyperdb.String()
         return d
 
-    def index(self, nodeid):
-        ''' Index the node in the search index.
-
-            We want to index the content in addition to the normal String
-            property indexing.
-        '''
-        # perform normal indexing
-        Class.index(self, nodeid)
-
-        # get the content to index
-        content = self.get(nodeid, 'content')
-
-        # figure the mime type
-        if self.properties.has_key('type'):
-            mime_type = self.get(nodeid, 'type')
-        else:
-            mime_type = self.default_mime_type
-
-        # and index!
-        self.db.indexer.add_text((self.classname, nodeid, 'content'), content,
-            mime_type)
 
 # deviation from spec - was called ItemClass
 class IssueClass(Class, roundupdb.IssueClass):

roundup/backends/back_metakit.py

@@ -1,4 +1,4 @@
-# $Id: back_metakit.py,v 1.62 2004-03-18 01:58:45 richard Exp $
+# $Id: back_metakit.py,v 1.63 2004-03-19 04:47:59 richard Exp $
 '''Metakit backend for Roundup, originally by Gordon McMillan.
 
 Known Current Bugs:
@@ -45,7 +45,7 @@
 import metakit
 from sessions_dbm import Sessions, OneTimeKeys
 import re, marshal, os, sys, time, calendar
-from roundup import indexer
+from indexer_dbm import Indexer
 import locking
 from roundup.date import Range
 
@@ -1783,7 +1783,7 @@ def __init__(self, db, classname, **properties):
 
 CURVERSION = 2
 
-class Indexer(indexer.Indexer):
+class Indexer(Indexer):
     disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1}
     def __init__(self, path, datadb):
         self.path = os.path.join(path, 'index.mk4')

roundup/backends/back_mysql.py

@@ -244,6 +244,7 @@ def drop_multilink_table_indexes(self, classname, ml):
             '%s_%s_l_idx'%(classname, ml),
             '%s_%s_n_idx'%(classname, ml)
         ]
+        table_name = '%s_%s'%(classname, ml)
         for index_name in l:
             if not self.sql_index_exists(table_name, index_name):
                 continue

roundup/backends/back_postgresql.py

@@ -146,30 +146,26 @@ def create_class_table(self, spec):
         cols, mls = self.determine_columns(spec.properties.items())
         cols.append('id')
         cols.append('__retired__')
-        scols = ',' . join(['"%s" VARCHAR(255)' % x for x in cols])
+        scols = ',' . join(['"%s" VARCHAR(255)'%x for x in cols])
         sql = 'CREATE TABLE "_%s" (%s)' % (spec.classname, scols)
-
         if __debug__:
-            print >>hyperdb.DEBUG, 'create_class', (self, sql)
-
+            print >>hyperdb.DEBUG, 'create_class_table', (self, sql)
         self.cursor.execute(sql)
         self.create_class_table_indexes(spec)
         return cols, mls
 
     def create_journal_table(self, spec):
-        cols = ',' . join(['"%s" VARCHAR(255)' % x
-                           for x in 'nodeid date tag action params' . split()])
+        cols = ',' . join(['"%s" VARCHAR(255)'%x
+          for x in 'nodeid date tag action params' . split()])
         sql  = 'CREATE TABLE "%s__journal" (%s)'%(spec.classname, cols)
-        
         if __debug__:
-            print >>hyperdb.DEBUG, 'create_class', (self, sql)
-
+            print >>hyperdb.DEBUG, 'create_journal_table', (self, sql)
         self.cursor.execute(sql)
         self.create_journal_table_indexes(spec)
 
     def create_multilink_table(self, spec, ml):
         sql = '''CREATE TABLE "%s_%s" (linkid VARCHAR(255),
-                   nodeid VARCHAR(255))''' % (spec.classname, ml)
+            nodeid VARCHAR(255))'''%(spec.classname, ml)
 
         if __debug__:
             print >>hyperdb.DEBUG, 'create_class', (self, sql)

roundup/backends/blobfiles.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-#$Id: blobfiles.py,v 1.11 2004-02-11 23:55:09 richard Exp $
+#$Id: blobfiles.py,v 1.12 2004-03-19 04:47:59 richard Exp $
 '''This module exports file storage for roundup backends.
 Files are stored into a directory hierarchy.
 '''
@@ -77,12 +77,14 @@ def storefile(self, classname, nodeid, property, content):
         if not os.path.exists(os.path.dirname(name)):
             os.makedirs(os.path.dirname(name))
 
-        # open the temp file for writing
-        open(name + '.tmp', 'wb').write(content)
-
-        # save off the commit action
-        self.transactions.append((self.doStoreFile, (classname, nodeid,
-            property)))
+        # save to a temp file
+        name = name + '.tmp'
+        # make sure we don't register the rename action more than once
+        if not os.path.exists(name):
+            # save off the rename action
+            self.transactions.append((self.doStoreFile, (classname, nodeid,
+                property)))
+        open(name, 'wb').write(content)
 
     def getfile(self, classname, nodeid, property):
         '''Get the content of the file in the database.
@@ -115,6 +117,11 @@ def doStoreFile(self, classname, nodeid, property, **databases):
         # determine the name of the file to write to
         name = self.filename(classname, nodeid, property)
 
+        # content is being updated (and some platforms, eg. win32, won't
+        # let us rename over the top of the old file)
+        if os.path.exists(name):
+            os.remove(name)
+
         # the file is currently ".tmp" - move it to its real name to commit
         os.rename(name+".tmp", name)
 

roundup/indexer.py roundup/backends/indexer_dbm.pyroundup/indexer.py renamed to roundup/backends/indexer_dbm.py

@@ -14,15 +14,15 @@
 #     that promote freedom, but obviously am giving up any rights
 #     to compel such.
 # 
-#$Id: indexer.py,v 1.18 2004-02-11 23:55:08 richard Exp $
+#$Id: indexer_dbm.py,v 1.1 2004-03-19 04:47:59 richard Exp $
 '''This module provides an indexer class, RoundupIndexer, that stores text
 indices in a roundup instance.  This class makes searching the content of
 messages, string properties and text files possible.
 '''
 __docformat__ = 'restructuredtext'
 
 import os, shutil, re, mimetypes, marshal, zlib, errno
-from hyperdb import Link, Multilink
+from roundup.hyperdb import Link, Multilink
 
 class Indexer:
     '''Indexes information from roundup's hyperdb to allow efficient
@@ -129,7 +129,7 @@ def text_splitter(self, text):
         """Split text/plain string into a list of words
         """
         # case insensitive
-        text = text.upper()
+        text = str(text).upper()
 
         # Split the raw text, losing anything longer than 25 characters
         # since that'll be gibberish (encoded text or somesuch) or shorter
@@ -341,4 +341,9 @@ def index_loaded(self):
         return (hasattr(self,'fileids') and hasattr(self,'files') and
             hasattr(self,'words'))
 
+
+    def rollback(self):
+        ''' load last saved index info. '''
+        self.load_index(reload=1)
+
 # vim: set filetype=python ts=4 sw=4 et si