Features and fixes.

Feature:
- trackers may configure custom stop-words for the full-text indexer

Fixed:
- fixes in scripts/import_sf.py
- fix some unicode bugs in roundup-admin import
- Xapian indexer wasn't actually being used
- fix indexing of message content on roundup-admin import

Author: Richard Jones

CHANGES.txt

@@ -1,9 +1,15 @@
 This file contains the changes to the Roundup system over time. The entries
 are given with the most recent entry first.
 
-2006-??-?? 1.0.2
+2006-02-06 1.0.2
+Feature:
+- trackers may configure custom stop-words for the full-text indexer
+
 Fixed:
-- silly typo in scripts/import_sf.py
+- fixes in scripts/import_sf.py
+- fix some unicode bugs in roundup-admin import
+- Xapian indexer wasn't actually being used
+- fix indexing of message content on roundup-admin import
 
 
 2006-02-03 1.0.1

doc/customizing.txt

@@ -2,7 +2,7 @@
 Customising Roundup
 ===================
 
-:Version: $Revision: 1.191 $
+:Version: $Revision: 1.192 $
 
 .. This document borrows from the ZopeBook section on ZPT. The original is at:
    http://www.zope.org/Documentation/Books/ZopeBook/current/ZPT.stx
@@ -140,6 +140,15 @@ Section **main**
   email?
   Allowed values: ``yes``, ``no``
 
+ email_registration_confirmation -- ``yes``
+  Offer registration confirmation by email or only through the web?
+  Allowed values: ``yes``, ``no``
+
+ indexer_stopwords -- default *blank*
+  Additional stop-words for the full-text indexer specific to
+  your tracker. See the indexer source for the default list of
+  stop-words (e.g. ``A,AND,ARE,AS,AT,BE,BUT,BY, ...``).
+
 Section **tracker**
  name -- ``Roundup issue tracker``
   A descriptive name for your roundup instance.

roundup/admin.py

@@ -16,7 +16,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 #
-# $Id: admin.py,v 1.97 2005-12-03 11:26:08 a1s Exp $
+# $Id: admin.py,v 1.98 2006-02-06 21:00:44 richard Exp $
 
 '''Administration commands for maintaining Roundup trackers.
 '''
@@ -1155,16 +1155,19 @@ class colon_separated(csv.excel):
             file_props = None
             maxid = 1
             # loop through the file and create a node for each entry
-            for r in reader:
+            for n, r in enumerate(reader):
                 if file_props is None:
                     file_props = r
                     continue
+                sys.stdout.write('Importing %s - %d\r'%(classname, n))
+                sys.stdout.flush()
+
                 # do the import and figure the current highest nodeid
                 nodeid = int(cl.import_list(file_props, r))
                 if hasattr(cl, 'import_files'):
                     cl.import_files(dir, nodeid)
                 maxid = max(maxid, nodeid)
-
+            print
             f.close()
 
             # import the journals
@@ -1238,7 +1241,7 @@ def do_reindex(self, args, desre=re.compile('([A-Za-z]+)([0-9]+)')):
                     cl = self.get_class(arg)
                     self.db.reindex(arg)
         else:
-            self.db.reindex()
+            self.db.reindex(show_progress=True)
         return 0
 
     def do_security(self, args):

roundup/backends/back_anydbm.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 #
-#$Id: back_anydbm.py,v 1.194 2006-01-24 08:26:09 a1s Exp $
+#$Id: back_anydbm.py,v 1.195 2006-02-06 21:00:46 richard Exp $
 '''This module defines a backend that saves the hyperdatabase in a
 database chosen by anydbm. It is guaranteed to always be available in python
 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
@@ -43,8 +43,7 @@
 from sessions_dbm import Sessions, OneTimeKeys
 
 try:
-    # re-enable once Xapian is fixed
-    from indexer_xapian import Indexer_disabled
+    from indexer_xapian import Indexer
 except ImportError:
     from indexer_dbm import Indexer
 

roundup/backends/back_metakit.py

@@ -1,4 +1,4 @@
-# $Id: back_metakit.py,v 1.103 2006-01-27 05:22:46 richard Exp $
+# $Id: back_metakit.py,v 1.104 2006-02-06 21:00:47 richard Exp $
 '''Metakit backend for Roundup, originally by Gordon McMillan.
 
 Known Current Bugs:
@@ -1816,6 +1816,14 @@ def import_files(self, dirname, nodeid):
             os.makedirs(os.path.dirname(dest))
         shutil.copyfile(source, dest)
 
+        mime_type = None
+        if self.getprops().has_key('type'):
+            mime_type = propvalues.get('type', self.get(itemid, 'type'))
+        if not mime_type:
+            mime_type = self.default_mime_type
+        self.db.indexer.add_text((self.classname, nodeid, 'content'),
+            self.get(nodeid, 'content'), mime_type)
+
     def get(self, nodeid, propname, default=_marker, cache=1):
         if propname == 'content':
             poss_msg = 'Possibly an access right configuration problem.'
@@ -2078,8 +2086,7 @@ def rollback(self):
         self.changed = 0
 
 try:
-    # re-enable once Xapian is fixed
-    from indexer_xapian import Indexer_disabled
+    from indexer_xapian import Indexer
 except ImportError:
     Indexer = MetakitIndexer
 

roundup/backends/indexer_common.py

@@ -1,26 +1,29 @@
-#$Id: indexer_common.py,v 1.4 2005-01-08 16:16:59 jlgijsbers Exp $
-import re
+#$Id: indexer_common.py,v 1.5 2006-02-06 21:00:47 richard Exp $
+import re, sets
 
 from roundup import hyperdb
 
-stopwords = [
-"A", "AND", "ARE", "AS", "AT", "BE", "BUT", "BY",
-"FOR", "IF", "IN", "INTO", "IS", "IT",
-"NO", "NOT", "OF", "ON", "OR", "SUCH",
-"THAT", "THE", "THEIR", "THEN", "THERE", "THESE",
-"THEY", "THIS", "TO", "WAS", "WILL", "WITH" 
+STOPWORDS = [
+    "A", "AND", "ARE", "AS", "AT", "BE", "BUT", "BY",
+    "FOR", "IF", "IN", "INTO", "IS", "IT",
+    "NO", "NOT", "OF", "ON", "OR", "SUCH",
+    "THAT", "THE", "THEIR", "THEN", "THERE", "THESE",
+    "THEY", "THIS", "TO", "WAS", "WILL", "WITH" 
 ]
 
-is_stopword = {}
-for word in stopwords:
-    is_stopword[word] = None
-is_stopword = is_stopword.has_key
-
 def _isLink(propclass):
     return (isinstance(propclass, hyperdb.Link) or
             isinstance(propclass, hyperdb.Multilink))
 
 class Indexer:    
+    def __init__(self, db):
+        self.stopwords = sets.Set(STOPWORDS)
+        for word in db.config[('main', 'indexer_stopwords')]:
+            self.stopwords.add(word)
+
+    def is_stopword(self, word):
+        return word in self.stopwords
+
     def getHits(self, search_terms, klass):
         return self.find(search_terms)
 

roundup/backends/indexer_dbm.py

@@ -14,7 +14,7 @@
 #     that promote freedom, but obviously am giving up any rights
 #     to compel such.
 # 
-#$Id: indexer_dbm.py,v 1.6 2005-04-28 00:21:42 richard Exp $
+#$Id: indexer_dbm.py,v 1.7 2006-02-06 21:00:47 richard Exp $
 '''This module provides an indexer class, RoundupIndexer, that stores text
 indices in a roundup instance.  This class makes searching the content of
 messages, string properties and text files possible.
@@ -23,9 +23,9 @@
 
 import os, shutil, re, mimetypes, marshal, zlib, errno
 from roundup.hyperdb import Link, Multilink
-from roundup.backends.indexer_common import Indexer, is_stopword
+from roundup.backends.indexer_common import Indexer as IndexerBase
 
-class Indexer(Indexer):
+class Indexer(IndexerBase):
     '''Indexes information from roundup's hyperdb to allow efficient
     searching.
 
@@ -38,6 +38,7 @@ class Indexer(Indexer):
     where identifier is (classname, nodeid, propertyname)
     '''
     def __init__(self, db):
+        IndexerBase.__init__(self, db)
         self.indexdb_path = os.path.join(db.config.DATABASE, 'indexes')
         self.indexdb = os.path.join(self.indexdb_path, 'index.db')
         self.reindex = 0
@@ -96,7 +97,7 @@ def add_text(self, identifier, text, mime_type='text/plain'):
         # find the unique words
         filedict = {}
         for word in words:
-            if is_stopword(word):
+            if self.is_stopword(word):
                 continue
             if filedict.has_key(word):
                 filedict[word] = filedict[word]+1

roundup/backends/indexer_rdbms.py

@@ -1,14 +1,15 @@
-#$Id: indexer_rdbms.py,v 1.11 2005-09-28 05:42:23 richard Exp $
+#$Id: indexer_rdbms.py,v 1.12 2006-02-06 21:00:47 richard Exp $
 ''' This implements the full-text indexer over two RDBMS tables. The first
 is a mapping of words to occurance IDs. The second maps the IDs to (Class,
 propname, itemid) instances.
 '''
 import re
 
-from indexer_common import Indexer, is_stopword
+from roundup.backends.indexer_common import Indexer as IndexerBase
 
-class Indexer(Indexer):
+class Indexer(IndexerBase):
     def __init__(self, db):
+        IndexerBase.__init__(self, db)
         self.db = db
         self.reindex = 0
 
@@ -62,7 +63,7 @@ def add_text(self, identifier, text, mime_type='text/plain'):
                 for w in re.findall(r'(?u)\b\w{2,25}\b', text)]
         words = {}
         for word in wordlist:
-            if is_stopword(word): continue
+            if self.is_stopword(word): continue
             if len(word) > 25: continue
             words[word] = 1
         words = words.keys()

roundup/backends/indexer_xapian.py

@@ -1,16 +1,17 @@
-#$Id: indexer_xapian.py,v 1.1 2005-04-28 00:21:42 richard Exp $
+#$Id: indexer_xapian.py,v 1.2 2006-02-06 21:00:47 richard Exp $
 ''' This implements the full-text indexer using the Xapian indexer.
 '''
 import re, os
 
 import xapian
 
-from indexer_common import Indexer, is_stopword
+from roundup.backends.indexer_common import Indexer as IndexerBase
 
 # TODO: we need to delete documents when a property is *reindexed*
 
-class Indexer(Indexer):
+class Indexer(IndexerBase):
     def __init__(self, db):
+        IndexerBase.__init__(self, db)
         self.db_path = db.config.DATABASE
         self.reindex = 0
         self.transaction_active = False
@@ -63,7 +64,7 @@ def add_text(self, identifier, text, mime_type='text/plain'):
         doc.set_data('%s:%s:%s'%identifier)
         for match in re.finditer(r'\b\w{2,25}\b', text.upper()):
             word = match.group(0)
-            if is_stopword(word):
+            if self.is_stopword(word):
                 continue
             term = stemmer.stem_word(word)
             doc.add_posting(term, match.start(0))

roundup/backends/rdbms_common.py

@@ -1,4 +1,4 @@
-# $Id: rdbms_common.py,v 1.164 2006-01-30 00:36:26 richard Exp $
+# $Id: rdbms_common.py,v 1.165 2006-02-06 21:00:47 richard Exp $
 ''' Relational database (SQL) backend common code.
 
 Basics:
@@ -32,16 +32,15 @@
 import sys, os, time, re, errno, weakref, copy, logging
 
 # roundup modules
-from roundup import hyperdb, date, password, roundupdb, security
+from roundup import hyperdb, date, password, roundupdb, security, support
 from roundup.hyperdb import String, Password, Date, Interval, Link, \
     Multilink, DatabaseError, Boolean, Number, Node
 from roundup.backends import locking
 
 # support
 from blobfiles import FileStorage
 try:
-    # re-enable once Xapian is fixed
-    from indexer_xapian import Indexer_disabled
+    from indexer_xapian import Indexer
 except ImportError:
     from indexer_rdbms import Indexer
 from sessions_rdbms import Sessions, OneTimeKeys
@@ -317,14 +316,19 @@ def refresh_database(self):
         self.post_init()
 
 
-    def reindex(self, classname=None):
+    def reindex(self, classname=None, show_progress=False):
         if classname:
             classes = [self.getclass(classname)]
         else:
             classes = self.classes.values()
         for klass in classes:
-            for nodeid in klass.list():
-                klass.index(nodeid)
+            if show_progress:
+                for nodeid in support.Progress('Reindex %s'%klass.classname,
+                        klass.list()):
+                    klass.index(nodeid)
+            else:
+                for nodeid in klass.list():
+                    klass.index(nodeid)
         self.indexer.save_index()
 
     hyperdb_to_sql_datatypes = {
@@ -2371,15 +2375,17 @@ def import_list(self, propnames, proplist):
                 pwd = password.Password()
                 pwd.unpack(value)
                 value = pwd
-            d[propname] = value
-            if isinstance(prop, String):
-                if type(value) != type('') and type(value) != type(u''):
+            elif isinstance(prop, String):
+                if isinstance(value, unicode):
+                    value = value.encode('utf8')
+                if not isinstance(value, str):
                     raise TypeError, \
                         'new property "%(propname)s" not a string: %(value)r' \
                         % locals()
                 if prop.indexme:
                     self.db.indexer.add_text((self.classname, newid, propname),
                         value)
+            d[propname] = value
 
         # get a new id if necessary
         if newid is None: