force sqlite3 in py2.6+ to treat our stored text as UTF-8

Author: Richard Jones

roundup/anypy/TODO.txt

@@ -22,6 +22,4 @@ Python compatiblity TODO
   MimeWriter.MimeWrite.addheader
   -> email.Message.Message.add_header     (2.3)
 
-- test.test_sqlite.sqliteDBTest.testStringUnicode fails
-
 # vim: si

roundup/backends/back_sqlite.py

@@ -108,6 +108,13 @@ def sql_open_connection(self):
         else:
             conn = sqlite.connect(db, timeout=30)
             conn.row_factory = sqlite.Row
+
+        # sqlite3 wants us to store Unicode in the db but that's not what's
+        # been done historically and it's definitely not what the other
+        # backends do, so we'll stick with UTF-8
+        if sqlite_version == 3:
+            conn.text_factory = str
+
         cursor = conn.cursor()
         return (conn, cursor)
 

roundup/backends/indexer_rdbms.py

@@ -65,8 +65,8 @@ def add_text(self, identifier, text, mime_type='text/plain'):
 
         # ok, find all the unique words in the text
         text = unicode(text, "utf-8", "replace").upper()
-        wordlist = [w.encode("utf-8", "replace")
-                for w in re.findall(r'(?u)\b\w{2,25}\b', text)]
+        wordlist = [w.encode("utf-8")
+            for w in re.findall(r'(?u)\b\w{2,25}\b', text)]
         words = set()
         for word in wordlist:
             if self.is_stopword(word): continue