Python 3 preparation: unicode.

This patch introduces roundup/anypy/strings.py, which has a comment
explaining the string representations generally used and common
functions to handle the required conversions.  Places in the code that
explicitly reference the "unicode" type / built-in function are
generally changed to use the new functions (or, in a few places where
those new functions don't seem to fit well, other approaches such as
references to type(u'') or use of the codecs module).  This patch does
not generally attempt to address text conversions in any places not
currently referencing the "unicode" type (although
scripts/import_sf.py is made to use binary I/O in places as fixing the
"unicode" reference didn't seem coherent otherwise).

Author: jsm28

roundup/anypy/strings.py

@@ -0,0 +1,75 @@
+# Roundup represents text internally using the native Python str type.
+# In Python 3, these are Unicode strings.  In Python 2, these are
+# encoded using UTF-8, and the Python 2 unicode type is only used in a
+# few places, generally for interacting with external modules
+# requiring that type to be used.
+
+import sys
+_py3 = sys.version_info[0] > 2
+
+def b2s(b):
+    """Convert a UTF-8 encoded bytes object to the internal string format."""
+    if _py3:
+        return b.decode('utf-8')
+    else:
+        return b
+
+def s2b(s):
+    """Convert a string object to UTF-8 encoded bytes."""
+    if _py3:
+        return s.encode('utf-8')
+    else:
+        return s
+
+def s2u(s, errors='strict'):
+    """Convert a string object to a Unicode string."""
+    if _py3:
+        return s
+    else:
+        return unicode(s, 'utf-8', errors)
+
+def u2s(u):
+    """Convert a Unicode string to the internal string format."""
+    if _py3:
+        return u
+    else:
+        return u.encode('utf-8')
+
+def us2u(s, errors='strict'):
+    """Convert a string or Unicode string to a Unicode string."""
+    if _py3:
+        return s
+    else:
+        if isinstance(s, unicode):
+            return s
+        else:
+            return unicode(s, 'utf-8', errors)
+
+def us2s(u):
+    """Convert a string or Unicode string to the internal string format."""
+    if _py3:
+        return u
+    else:
+        if isinstance(u, unicode):
+            return u.encode('utf-8')
+        else:
+            return u
+
+def uany2s(u):
+    """Convert a Unicode string or other object to the internal string format.
+
+    Objects that are not Unicode strings are passed to str()."""
+    if _py3:
+        return str(u)
+    else:
+        if isinstance(u, unicode):
+            return u.encode('utf-8')
+        else:
+            return str(u)
+
+def is_us(s):
+    """Return whether an object is a string or Unicode string."""
+    if _py3:
+        return isinstance(s, str)
+    else:
+        return isinstance(s, str) or isinstance(s, unicode)

roundup/backends/back_sqlite.py

@@ -13,6 +13,7 @@
 from roundup import hyperdb, date, password
 from roundup.backends import rdbms_common
 from roundup.backends.sessions_dbm import Sessions, OneTimeKeys
+from roundup.anypy.strings import uany2s
 
 sqlite_version = None
 try:
@@ -85,7 +86,7 @@ class Database(rdbms_common.Database):
         hyperdb.Multilink : lambda x: x,    # used in journal marshalling
     }
     sql_to_hyperdb_value = {
-        hyperdb.String : lambda x: isinstance(x, unicode) and x.encode('utf8') or str(x),
+        hyperdb.String : uany2s,
         hyperdb.Date   : lambda x: date.Date(str(x)),
         hyperdb.Link   : str, # XXX numeric ids
         hyperdb.Interval  : date.Interval,

roundup/backends/indexer_rdbms.py

@@ -5,6 +5,7 @@
 import re
 
 from roundup.backends.indexer_common import Indexer as IndexerBase
+from roundup.anypy.strings import us2u, u2s
 
 class Indexer(IndexerBase):
     def __init__(self, db):
@@ -61,10 +62,9 @@ def add_text(self, identifier, text, mime_type='text/plain'):
             self.db.cursor.execute(sql, (id, ))
 
         # ok, find all the unique words in the text
-        if not isinstance(text, unicode):
-            text = unicode(text, "utf-8", "replace")
+        text = us2u(text, "replace")
         text = text.upper()
-        wordlist = [w.encode("utf-8")
+        wordlist = [u2s(w)
                     for w in re.findall(r'(?u)\b\w{%d,%d}\b'
                                         % (self.minlength, self.maxlength), text)]
         words = set()

roundup/backends/indexer_whoosh.py

@@ -5,6 +5,7 @@
 from whoosh import fields, qparser, index, query, analysis
 
 from roundup.backends.indexer_common import Indexer as IndexerBase
+from roundup.anypy.strings import us2u
 
 class Indexer(IndexerBase):
     def __init__(self, db):
@@ -78,8 +79,7 @@ def add_text(self, identifier, text, mime_type='text/plain'):
         if not text:
             text = u''
 
-        if not isinstance(text, unicode):
-            text = unicode(text, "utf-8", "replace")
+        text = us2u(text, "replace")
 
         # We use the identifier twice: once in the actual "text" being
         # indexed so we can search on it, and again as the "data" being

roundup/backends/rdbms_common.py

@@ -69,6 +69,7 @@
 from roundup.date import Range
 
 from roundup.backends.back_anydbm import compile_expression
+from roundup.anypy.strings import us2s
 
 
 # dummy value meaning "argument not passed"
@@ -2944,8 +2945,7 @@ def import_list(self, propnames, proplist):
             elif isinstance(prop, hyperdb.Password):
                 value = password.Password(encrypted=value)
             elif isinstance(prop, String):
-                if isinstance(value, unicode):
-                    value = value.encode('utf8')
+                value = us2s(value)
                 if not isinstance(value, str):
                     raise TypeError('new property "%(propname)s" not a '
                         'string: %(value)r'%locals())

roundup/cgi/PageTemplates/TALES.py

@@ -231,7 +231,7 @@ def evaluateText(self, expr):
         text = self.evaluate(expr)
         if text is Default or text is None:
             return text
-        if isinstance(text, unicode):
+        if isinstance(text, type(u'')):
             return text
         else:
             return ustr(text)

roundup/cgi/TranslationService.py

@@ -16,13 +16,12 @@
 from roundup import i18n
 from roundup.cgi.PageTemplates import Expressions, PathIterator, TALES
 from roundup.cgi.TAL import TALInterpreter
+from roundup.anypy.strings import us2u, u2s
 
 ### Translation classes
 
 class TranslationServiceMixin:
 
-    OUTPUT_ENCODING = "utf-8"
-
     def translate(self, domain, msgid, mapping=None,
         context=None, target_language=None, default=None
     ):
@@ -32,18 +31,15 @@ def translate(self, domain, msgid, mapping=None,
         return _msg
 
     def gettext(self, msgid):
-        if not isinstance(msgid, unicode):
-            msgid = unicode(msgid, 'utf8')
+        msgid = us2u(msgid)
         msgtrans=self.ugettext(msgid)
-        return msgtrans.encode(self.OUTPUT_ENCODING)
+        return u2s(msgtrans)
 
     def ngettext(self, singular, plural, number):
-        if not isinstance(singular, unicode):
-            singular = unicode(singular, 'utf8')
-        if not isinstance(plural, unicode):
-            plural = unicode(plural, 'utf8')
+        singular = us2u(singular)
+        plural = us2u(plural)
         msgtrans=self.ungettext(singular, plural, number)
-        return msgtrans.encode(self.OUTPUT_ENCODING)
+        return u2s(msgtrans)
 
 class TranslationService(TranslationServiceMixin, i18n.RoundupTranslations):
     pass
@@ -55,8 +51,7 @@ def ugettext(self, message):
             return self._fallback.ugettext(message)
         # Sometimes the untranslatable message is a UTF-8 encoded string
         # (thanks to PageTemplate's internals).
-        if not isinstance(message, unicode):
-            return unicode(message, 'utf8')
+        message = us2u(message)
         return message
 
 ### TAL patching

roundup/cgi/engine_chameleon.py

@@ -6,6 +6,7 @@
 import chameleon
 
 from roundup.cgi.templating import StringIO, context, TALLoaderBase
+from roundup.anypy.strings import s2u
 
 class Loader(TALLoaderBase):
     def __init__(self, dir):
@@ -27,7 +28,7 @@ def render(self, client, classname, request, **options):
         def translate(msgid, domain=None, mapping=None, default=None):
             result = client.translator.translate(domain, msgid,
                          mapping=mapping, default=default)
-            return unicode(result, client.translator.OUTPUT_ENCODING)
+            return s2u(result)
 
         output = self._pt.render(None, translate, **c)
         return output.encode(client.charset)

roundup/cgi/engine_jinja2.py

@@ -40,6 +40,7 @@
 # http://jinja.pocoo.org/docs/api/#loaders
 
 from roundup.cgi.templating import context, LoaderBase, TemplateBase
+from roundup.anypy.strings import s2u
 
 class Jinja2Loader(LoaderBase):
     def __init__(self, dir):
@@ -59,8 +60,7 @@ def __init__(self, dir):
         # The automatic conversion will assume 'ascii' and fail sometime.
         # Analysed with roundup 1.5.0 and jinja 2.7.1. See issue2550811.
         self._env.filters["u"] = lambda s: \
-            unicode(s(), "utf-8") if type(s) == MethodType \
-                                  else unicode(s, "utf-8")
+            s2u(s()) if type(s) == MethodType else s2u(s)
 
     def check(self, tplname):
         #print tplname

roundup/cgi/templating.py

@@ -29,6 +29,7 @@
 from roundup import hyperdb, date, support
 from roundup import i18n
 from roundup.i18n import _
+from roundup.anypy.strings import is_us, us2s, s2u, u2s
 
 from .KeywordsExpr import render_keywords_expression_editor
 
@@ -1774,7 +1775,7 @@ def field(self, labelfirst=False, y_label=None, n_label=None,
             return self.plain(escape=1)
 
         value = self._value
-        if isinstance(value, str) or isinstance(value, unicode):
+        if is_us(value):
             value = value.strip().lower() in ('checked', 'yes', 'true',
                 'on', '1')
 
@@ -1827,8 +1828,7 @@ def __init__(self, client, classname, nodeid, prop, name, value,
             anonymous=0, offset=None):
         HTMLProperty.__init__(self, client, classname, nodeid, prop, name,
                 value, anonymous=anonymous)
-        if self._value and not (isinstance(self._value, str) or
-                isinstance(self._value, unicode)):
+        if self._value and not is_us(self._value):
             self._value.setTranslator(self._client.translator)
         self._offset = offset
         if self._offset is None :
@@ -1910,9 +1910,9 @@ def field(self, size=30, default=None, format=_marker, popcal=True,
                     raise ValueError(self._('default value for '
                         'DateHTMLProperty must be either DateHTMLProperty '
                         'or string date representation.'))
-        elif isinstance(value, str) or isinstance(value, unicode):
+        elif is_us(value):
             # most likely erroneous input to be passed back to user
-            if isinstance(value, unicode): value = value.encode('utf8')
+            value = us2s(value)
             s = self.input(name=self._formname, value=value, size=size,
                               **kwargs)
             if popcal:
@@ -1923,7 +1923,7 @@ def field(self, size=30, default=None, format=_marker, popcal=True,
 
         if raw_value is None:
             value = ''
-        elif isinstance(raw_value, str) or isinstance(raw_value, unicode):
+        elif is_us(raw_value):
             if format is self._marker:
                 value = raw_value
             else:
@@ -2012,7 +2012,7 @@ def __init__(self, client, classname, nodeid, prop, name, value,
             anonymous=0):
         HTMLProperty.__init__(self, client, classname, nodeid, prop,
             name, value, anonymous)
-        if self._value and not isinstance(self._value, (str, unicode)):
+        if self._value and not is_us(self._value):
             self._value.setTranslator(self._client.translator)
 
     def plain(self, escape=0):
@@ -2967,9 +2967,9 @@ def batch(self, permission='View'):
         klass = self.client.db.getclass(self.classname)
         if self.search_text:
             matches = self.client.db.indexer.search(
-                [w.upper().encode("utf-8", "replace") for w in re.findall(
+                [u2s(w.upper()) for w in re.findall(
                     r'(?u)\b\w{2,25}\b',
-                    unicode(self.search_text, "utf-8", "replace")
+                    s2u(self.search_text, "replace")
                 )], klass)
         else:
             matches = None