fix reindexing in Xapian

Author: Richard Jones

CHANGES.txt

@@ -9,6 +9,7 @@ Fixed:
 - fixes in scripts/import_sf.py
 - fix some unicode bugs in roundup-admin import
 - Xapian indexer wasn't actually being used
+- fix reindexing in Xapian
 - fix indexing of message content on roundup-admin import
 
 

roundup/__init__.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 # 
-# $Id: __init__.py,v 1.34 2006-02-03 04:12:36 richard Exp $
+# $Id: __init__.py,v 1.35 2006-02-07 04:59:05 richard Exp $
 
 '''Roundup - issue tracking for knowledge workers.
 
@@ -68,6 +68,6 @@
 '''
 __docformat__ = 'restructuredtext'
 
-__version__ = '1.0.1'
+__version__ = '1.0.2p1'
 
 # vim: set filetype=python ts=4 sw=4 et si

roundup/backends/indexer_xapian.py

@@ -1,4 +1,4 @@
-#$Id: indexer_xapian.py,v 1.2 2006-02-06 21:00:47 richard Exp $
+#$Id: indexer_xapian.py,v 1.3 2006-02-07 04:59:05 richard Exp $
 ''' This implements the full-text indexer using the Xapian indexer.
 '''
 import re, os
@@ -47,6 +47,7 @@ def add_text(self, identifier, text, mime_type='text/plain'):
         ''' "identifier" is  (classname, itemid, property) '''
         if mime_type != 'text/plain':
             return
+        if not text: text = ''
 
         # open the database and start a transaction if needed
         database = self._get_database()
@@ -57,18 +58,38 @@ def add_text(self, identifier, text, mime_type='text/plain'):
 
         # TODO: allow configuration of other languages
         stemmer = xapian.Stem("english")
+
+        # We use the identifier twice: once in the actual "text" being
+        # indexed so we can search on it, and again as the "data" being
+        # indexed so we know what we're matching when we get results
+        identifier = '%s:%s:%s'%identifier
+
+        # see if the id is in the database
+        enquire = xapian.Enquire(database)
+        query = xapian.Query(xapian.Query.OP_AND, [identifier])
+        enquire.set_query(query)
+        matches = enquire.get_mset(0, 10)
+        if matches.size():      # would it killya to implement __len__()??
+            b = matches.begin()
+            docid = b.get_docid()
+        else:
+            docid = None
+
+        # create the new document
         doc = xapian.Document()
+        doc.set_data(identifier)
+        doc.add_posting(identifier, 0)
 
-        # Xapian doesn't actually seem to care what data is put in here, so
-        # we use it to store the text identifier.
-        doc.set_data('%s:%s:%s'%identifier)
         for match in re.finditer(r'\b\w{2,25}\b', text.upper()):
             word = match.group(0)
             if self.is_stopword(word):
                 continue
             term = stemmer.stem_word(word)
             doc.add_posting(term, match.start(0))
-        database.add_document(doc)
+        if docid:
+            database.replace_document(docid, doc)
+        else:
+            database.add_document(doc)
 
     def find(self, wordlist):
         '''look up all the words in the wordlist.

test/db_test_base.py

@@ -15,7 +15,7 @@
 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 #
-# $Id: db_test_base.py,v 1.65 2006-02-07 04:14:32 richard Exp $
+# $Id: db_test_base.py,v 1.66 2006-02-07 04:59:05 richard Exp $
 
 import unittest, os, shutil, errno, imp, sys, time, pprint, sets
 
@@ -735,7 +735,7 @@ def testIndexerSearching(self):
         # unindexed stopword
         self.assertEquals(self.db.indexer.search(['the'], self.db.issue), {})
 
-    def testReindexing(self):
+    def testReindexingChange(self):
         search = self.db.indexer.search
         issue = self.db.issue
         i1 = issue.create(title="flebble plop")
@@ -750,6 +750,15 @@ def testReindexing(self):
         self.assertEquals(search(['plop'], issue), {i1: {}})
         self.assertEquals(search(['flebble'], issue), {i2: {}})
 
+    def testReindexingClear(self):
+        search = self.db.indexer.search
+        issue = self.db.issue
+        i1 = issue.create(title="flebble plop")
+        i2 = issue.create(title="flebble frooz")
+        self.db.commit()
+        self.assertEquals(search(['plop'], issue), {i1: {}})
+        self.assertEquals(search(['flebble'], issue), {i1: {}, i2: {}})
+
         # unset i1's title
         issue.set(i1, title="")
         self.db.commit()

test/test_indexer.py

@@ -18,7 +18,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-# $Id: test_indexer.py,v 1.9 2006-02-07 04:14:32 richard Exp $
+# $Id: test_indexer.py,v 1.10 2006-02-07 04:59:05 richard Exp $
 
 import os, unittest, shutil
 
@@ -46,10 +46,22 @@ def test_basics(self):
         self.assertEqual(self.dex.find(['blah']), [('test', '2', 'foo')])
         self.assertEqual(self.dex.find(['blah', 'hello']), [])
 
-        # change
+    def test_change(self):
+        self.dex.add_text(('test', '1', 'foo'), 'a the hello world')
+        self.dex.add_text(('test', '2', 'foo'), 'blah blah the world')
+        self.assertEqual(self.dex.find(['world']), [('test', '1', 'foo'),
+                                                    ('test', '2', 'foo')])
         self.dex.add_text(('test', '1', 'foo'), 'a the hello')
         self.assertEqual(self.dex.find(['world']), [('test', '2', 'foo')])
 
+    def test_clear(self):
+        self.dex.add_text(('test', '1', 'foo'), 'a the hello world')
+        self.dex.add_text(('test', '2', 'foo'), 'blah blah the world')
+        self.assertEqual(self.dex.find(['world']), [('test', '1', 'foo'),
+                                                    ('test', '2', 'foo')])
+        self.dex.add_text(('test', '1', 'foo'), '')
+        self.assertEqual(self.dex.find(['world']), [('test', '2', 'foo')])
+
     def tearDown(self):
         shutil.rmtree('test-index')