Replaced the use of unaccent.asciify(), which has similar functionality to unidecode.unidecode(). Changed the draft parser to work exclusively with unicode text, which both makes the removal of unaccent easier, and takes us closer to Py35 compatibility. Adjusted callers of the draft parser to send in unicode.

 - Legacy-Id: 13673

Author: levkowetz

ietf/doc/utils.py

@@ -498,9 +498,10 @@ def rebuild_reference_relations(doc,filename=None):
             filename=os.path.join(settings.INTERNET_DRAFT_PATH,doc.filename_with_rev())
 
     try:
-       refs = draft.Draft(draft._gettext(filename), filename).get_refs()
+        with open(filename, 'rb') as file:
+            refs = draft.Draft(file.read().decode('utf8'), filename).get_refs()
     except IOError as e:
-       return { 'errors': ["%s :%s" %  (e.strerror, filename)] }
+        return { 'errors': ["%s :%s" %  (e.strerror, filename)] }
 
     doc.relateddocument_set.filter(relationship__slug__in=['refnorm','refinfo','refold','refunk']).delete()
 

ietf/nomcom/utils.py

@@ -20,7 +20,7 @@
 from ietf.person.models import Email, Person
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.utils.pipe import pipe
-from ietf.utils import unaccent
+from unidecode import unidecode
 from ietf.utils.mail import send_mail_text, send_mail
 from ietf.utils.log import log
 
@@ -365,7 +365,7 @@ def make_nomineeposition_for_newperson(nomcom, candidate_name, candidate_email,
     # This is expected to fail if called with an existing email address
     email = Email.objects.create(address=candidate_email)
     person = Person.objects.create(name=candidate_name,
-                                   ascii=unaccent.asciify(candidate_name),
+                                   ascii=unidecode(candidate_name),
                                    address=candidate_email)
     email.person = person
     email.save()

ietf/secr/drafts/views.py

@@ -108,7 +108,7 @@ def process_files(request,draft):
         file_type_list.append(extension)
         if extension == '.txt':
             txt_size = file.size
-            wrapper = Draft(file.read(),file.name)
+            wrapper = Draft(file.read().decode('utf8'),file.name)
         handle_uploaded_file(file)
 
     # create Submission record, leaved unsaved

ietf/stats/backfill_data.py

@@ -6,6 +6,7 @@
 import os
 import os.path
 import argparse
+import six
 import time
 
 basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
@@ -59,13 +60,14 @@ def unicode(text):
     for encoding in ['ascii', 'utf8', 'latin1', ]:
         try:
             utext = text.decode(encoding) 
-            if encoding == 'latin1':
-                say("Warning: falling back to latin1 decoding for %s" % utext)
+#             if encoding == 'latin1':
+#                 say("Warning: falling back to latin1 decoding for %s ..." % utext[:216]])
             return utext
         except UnicodeDecodeError:
             pass
 
 start = time.time()
+say("Running query for documents to process ...")
 for doc in docs_qs.prefetch_related("docalias_set", "formal_languages", "documentauthor_set", "documentauthor_set__person", "documentauthor_set__person__alias_set"):
     canonical_name = doc.name
     for n in doc.docalias_set.all():
@@ -81,10 +83,10 @@ def unicode(text):
         say("Skipping %s, no txt file found at %s" % (doc.name, path))
         continue
 
-    with open(path, 'r') as f:
+    with open(path, 'rb') as f:
         say("\nProcessing %s" % doc.name)
         sys.stdout.flush()
-        d = Draft(f.read(), path)
+        d = Draft(unicode(f.read()), path)
 
         updated = False
 
@@ -126,7 +128,11 @@ def unicode(text):
             # it's an extra author - skip those extra authors
             seen = set()
             for full, _, _, _, _, email, country, company in d.get_author_list():
-                full, email, country, company = [ unicode(s) for s in [full, email, country, company, ] ]
+                assert full is None or    isinstance(full,    six.text_type)
+                assert email is None or   isinstance(email,   six.text_type)
+                assert country is None or isinstance(country, six.text_type)
+                assert company is None or isinstance(company, six.text_type)
+                #full, email, country, company = [ unicode(s) for s in [full, email, country, company, ] ]
                 if email in seen:
                     continue
                 seen.add(email)

ietf/submit/forms.py

@@ -203,7 +203,7 @@ def clean(self):
             # try to parse it
             txt_file = self.cleaned_data['txt']
             txt_file.seek(0)
-            self.parsed_draft = Draft(txt_file.read(), txt_file.name)
+            self.parsed_draft = Draft(txt_file.read().decode('utf8'), txt_file.name)
             self.filename = self.parsed_draft.filename
             self.revision = self.parsed_draft.revision
             self.title    = self.parsed_draft.get_title()

ietf/submit/tests.py

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+
+
 import datetime
 import os
 import shutil
@@ -302,11 +304,11 @@ def submit_existing(self, formats, change_authors=True, group_type='wg', stream_
             draft.save_with_history([DocEvent.objects.create(doc=draft, rev=draft.rev, type="added_comment", by=Person.objects.get(user__username="secretary"), desc="Test")])
         if not change_authors:
             draft.documentauthor_set.all().delete()
-            author_person, author_email = ensure_person_email_info_exists('Author Name','author@example.com')
+            author_person, author_email = ensure_person_email_info_exists(u'Author Name',u'author@example.com')
             draft.documentauthor_set.create(person=author_person, email=author_email)
         else:
             # Make it such that one of the previous authors has an invalid email address
-            bogus_person, bogus_email = ensure_person_email_info_exists('Bogus Person',None)
+            bogus_person, bogus_email = ensure_person_email_info_exists(u'Bogus Person',None)
             DocumentAuthor.objects.create(document=draft, person=bogus_person, email=bogus_email, order=draft.documentauthor_set.latest('order').order+1)
 
         prev_author = draft.documentauthor_set.all()[0]

ietf/submit/utils.py

@@ -2,7 +2,7 @@
 
 import os
 import datetime
-import six
+import six                              # pyflakes:ignore
 from unidecode import unidecode
 
 from django.conf import settings
@@ -24,7 +24,6 @@
 from ietf.submit.mail import announce_to_lists, announce_new_version, announce_to_authors
 from ietf.submit.models import Submission, SubmissionEvent, Preapproval, DraftSubmissionStateName
 from ietf.utils import log
-from ietf.utils import unaccent
 from ietf.utils.mail import is_valid_email
 
 
@@ -401,10 +400,8 @@ def ensure_person_email_info_exists(name, email):
     if not person:
         person = Person()
         person.name = name
-        if isinstance(person.name, six.text_type):
-            person.ascii = unidecode(person.name).decode('ascii')
-        else:
-            person.ascii = unaccent.asciify(person.name).decode('ascii')
+        log.assertion('isinstance(person.name, six.text_type)')
+        person.ascii = unidecode(person.name).decode('ascii')
         person.save()
 
     # make sure we have an email address

ietf/submit/views.py

@@ -71,7 +71,7 @@ def upload_submission(request):
                     # be retrieved from the generated text file.  Provide a
                     # parsed draft object to get at that kind of information.
                     with open(file_name['txt']) as txt_file:
-                        form.parsed_draft = Draft(txt_file.read(), txt_file.name)
+                        form.parsed_draft = Draft(txt_file.read().decode('utf8'), txt_file.name)
 
                 else:
                     file_size = form.cleaned_data['txt'].size

ietf/utils/draft.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # -*- python -*-
-
+from __future__ import unicode_literals
 """
 NAME
 	%(program)s - Extract meta-information from an IETF draft.
@@ -37,9 +37,11 @@
 import os.path
 import re
 import stat
+import six
 import sys
 import time
 
+
 version = "0.35"
 program = os.path.basename(sys.argv[0])
 progdir = os.path.dirname(sys.argv[0])
@@ -124,6 +126,7 @@ def acronym_match(s, l):
 class Draft():
 
     def __init__(self, text, source):
+        assert isinstance(text, six.text_type)
         self.source = source
         self.rawtext = text
 
@@ -1168,8 +1171,8 @@ def getmeta(fn):
         return
 
     timestamp = time.strftime("%Y-%m-%dT%H:%M:%S+00:00", time.gmtime(os.stat(filename)[stat.ST_MTIME]))
-    text = _gettext(filename)
-    draft = Draft(text, filename)
+    with open(filename, 'rb') as file:
+        draft = Draft(file.read().decode('utf8'), filename)
     #_debug("\n".join(draft.lines))
 
     fields["eventdate"] = timestamp