Third part of document read refactoring, after [14406] and [14410]. This replaces all usage of the non-unicode-aware get_document_content() function with unicode-aware Document.text() or Document.text_or_error() methods. This was triggered by yet another report of unicode content not being shown properly, and should fix all instances of document (drafts, agendas, minutes, etc.) display in the datatracker not handling unicode characters properly.

 - Legacy-Id: 14411
Note: SVN reference [14406] has been migrated to Git commit 967ece7

Note: SVN reference [14410] has been migrated to Git commit 660c81c

Author: levkowetz

ietf/doc/mails.py

@@ -1,6 +1,5 @@
 # generation of mails 
 
-import os
 import textwrap, datetime
 
 from django.template.loader import render_to_string
@@ -13,11 +12,11 @@
 from ietf.utils.mail import send_mail, send_mail_text
 from ietf.ipr.utils import iprs_from_docs, related_docs
 from ietf.doc.models import WriteupDocEvent, LastCallDocEvent, DocAlias, ConsensusDocEvent
-from ietf.doc.utils import needed_ballot_positions, get_document_content
+from ietf.doc.utils import needed_ballot_positions
 from ietf.group.models import Role
 from ietf.doc.models import Document
 from ietf.mailtrigger.utils import gather_address_lists
-from ietf.utils import log
+
 
 def email_state_changed(request, doc, text, mailtrigger_id=None):
     (to,cc) = gather_address_lists(mailtrigger_id or 'doc_state_edited',doc=doc)
@@ -512,18 +511,7 @@ def send_review_possibly_replaces_request(request, doc, submitter_info):
 
 def email_charter_internal_review(request, charter):
     addrs = gather_address_lists('charter_internal_review',doc=charter,group=charter.group)
-    filename = '%s-%s.txt' % (charter.canonical_name(),charter.rev)
-    charter_text = get_document_content(
-                        filename,
-                        os.path.join(settings.CHARTER_PATH,filename),
-                        split=False,
-                        markup=False,
-                   )
-    utext = charter.text_or_error()     # pyflakes:ignore
-    if charter_text and charter_text != utext and not 'Error; cannot read' in charter_text:
-        debug.show('charter_text[:64]')
-        debug.show('utext[:64]')
-        log.assertion('charter_text == utext')
+    charter_text = charter.text_or_error()     # pyflakes:ignore
 
     send_mail(request, addrs.to, settings.DEFAULT_FROM_EMAIL,
               'Internal %s Review: %s (%s)'%(charter.group.type.name,charter.group.name,charter.group.acronym),

ietf/doc/models.py

@@ -437,6 +437,10 @@ def replaced_by(self):
 
     def text(self):
         path = self.get_file_name()
+        root, ext =  os.path.splitext(path)
+        txtpath = root+'.txt'
+        if ext != '.txt' and os.path.exists(txtpath):
+            path = txtpath
         try:
             with open(path, 'rb') as file:
                 raw = file.read()
@@ -450,7 +454,7 @@ def text(self):
         return text
 
     def text_or_error(self):
-        return self.text() or "Error; cannot read (%s)"%self.get_file_name()
+        return self.text() or "Error; cannot read '%s'"%self.get_base_name()
 
     def htmlized(self):
         name = self.get_base_name()
@@ -607,7 +611,7 @@ def canonical_name(self):
         if not hasattr(self, '_canonical_name'):
             name = self.name
             if self.type_id == "draft" and self.get_state_slug() == "rfc":
-                a = self.docalias_set.filter(name__startswith="rfc").first()
+                a = self.docalias_set.filter(name__startswith="rfc").order_by('-name').first()
                 if a:
                     name = a.name
             elif self.type_id == "charter":

ietf/doc/templatetags/ietf_filters.py

@@ -2,7 +2,6 @@
 
 import re
 import datetime
-import os
 import types
 from email.utils import parseaddr
 
@@ -16,9 +15,7 @@
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import ConsensusDocEvent
-from ietf.doc.utils import get_document_content
 from ietf.utils.text import wordwrap, fill, wrap_text_if_unwrapped
-from ietf.utils import log
 
 register = template.Library()
 
@@ -508,13 +505,7 @@ def emailwrap(email):
 def document_content(doc):
     if doc is None:
         return None
-    path = os.path.join(doc.get_file_path(),doc.filename_with_rev())
-    content = get_document_content(doc.name,path,markup=False)
-    utext = doc.text_or_error()         # pyflakes:ignore
-    if content and content != utext and not 'Error; cannot read' in content:
-        debug.show('content[:64]')
-        debug.show('utext[:64]')
-        log.assertion('content == utext')
+    content = doc.text_or_error()         # pyflakes:ignore
     return content
 
 @register.filter

ietf/doc/utils.py

@@ -25,6 +25,7 @@
 from ietf.utils import draft, text
 from ietf.utils.mail import send_mail
 from ietf.mailtrigger.utils import gather_address_lists
+from ietf.utils import log
 
 def save_document_in_history(doc):
     """Save a snapshot of document and related objects in the database."""
@@ -299,7 +300,7 @@ def get_unicode_document_content(key, filename, codec='utf-8', errors='ignore'):
     return raw_content
 
 def get_document_content(key, filename, split=True, markup=True):
-    #log.unreachable("2017-12-05")
+    log.unreachable("2017-12-05")
     try:
         with open(filename, 'rb') as f:
             raw_content = f.read()

ietf/doc/views_conflict_review.py

@@ -12,7 +12,7 @@
 from ietf.doc.models import ( BallotDocEvent, BallotPositionDocEvent, DocAlias, DocEvent,
     Document, NewRevisionDocEvent, State )
 from ietf.doc.utils import ( add_state_change_event, close_open_ballots,
-    create_ballot_if_not_open, get_document_content, update_telechat )
+    create_ballot_if_not_open, update_telechat )
 from ietf.doc.mails import email_iana
 from ietf.doc.forms import AdForm 
 from ietf.group.models import Role, Group
@@ -253,14 +253,7 @@ def edit_ad(request, name):
 
 def default_approval_text(review):
 
-    filename = "%s-%s.txt" % (review.canonical_name(), review.rev)
-    current_text = get_document_content(filename, os.path.join(settings.CONFLICT_REVIEW_PATH, filename), split=False, markup=False)
-    utext = review.text_or_error()      # pyflakes:ignore
-    if current_text and current_text != utext and not 'Error; cannot read' in current_text:
-        debug.show('current_text[:64]')
-        debug.show('utext[:64]')
-        log.assertion('current_text == utext')
-
+    current_text = review.text_or_error()      # pyflakes:ignore
     conflictdoc = review.relateddocument_set.get(relationship__slug='conflrev').target.document
     if conflictdoc.stream_id=='ise':
          receiver = 'Independent Submissions Editor'

ietf/doc/views_doc.py

@@ -47,7 +47,7 @@
     ConsensusDocEvent, NewRevisionDocEvent, TelechatDocEvent, WriteupDocEvent,
     IESG_BALLOT_ACTIVE_STATES, STATUSCHANGE_RELATIONS )
 from ietf.doc.utils import ( add_links_in_new_revision_events, augment_events_with_revision,
-    can_adopt_draft, get_chartering_type, get_document_content, get_tags_for_stream_id,
+    can_adopt_draft, get_chartering_type, get_tags_for_stream_id,
     needed_ballot_positions, nice_consensus, prettify_std_name, update_telechat, has_same_ballot,
     get_initial_notify, make_notify_changed_event, make_rev_history, default_consensus,
     add_events_message_info, get_unicode_document_content, build_doc_meta_block)
@@ -66,7 +66,7 @@
 from ietf.review.models import ReviewRequest
 from ietf.review.utils import can_request_review_of_doc, review_requests_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
-from ietf.utils import markup_txt, log
+from ietf.utils import markup_txt
 from ietf.utils.text import maybe_split
 
 
@@ -185,15 +185,7 @@ def document_main(request, name, rev=None):
 
         if doc.get_state_slug() == "rfc":
             # content
-            filename = name + ".txt"
-
-            content = get_document_content(filename, os.path.join(settings.RFC_PATH, filename),
-                                           split_content, markup=True)
-            utext = doc.text_or_error() # pyflakes:ignore
-            if content and content != utext and not 'Error; cannot read' in content:
-                debug.show('content[:64]')
-                debug.show('utext[:64]')
-                log.assertion('content == utext')
+            content = doc.text_or_error() # pyflakes:ignore
             content = markup_txt.markup(maybe_split(content, split=split_content))
 
             # file types
@@ -221,15 +213,7 @@ def document_main(request, name, rev=None):
                 content = "This RFC is not available in plain text format."
                 split_content = False
         else:
-            filename = "%s-%s.txt" % (draft_name, doc.rev)
-
-            content = get_document_content(filename, os.path.join(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR, filename),
-                                           split_content, markup=True)
-            utext = doc.text_or_error() # pyflakes:ignore
-            if content and content != utext and not 'Error; cannot read' in content:
-                debug.show('content[:64]')
-                debug.show('utext[:64]')
-                log.assertion('content == utext')
+            content = doc.text_or_error() # pyflakes:ignore
             content = markup_txt.markup(maybe_split(content, split=split_content)) 
 
             # file types
@@ -451,14 +435,7 @@ def document_main(request, name, rev=None):
                                        ))
 
     if doc.type_id == "charter":
-        filename = "%s-%s.txt" % (doc.canonical_name(), doc.rev)
-
-        content = get_document_content(filename, os.path.join(settings.CHARTER_PATH, filename), split=False, markup=True)
-        utext = doc.text_or_error()     # pyflakes:ignore
-        if content and content != utext and not 'Error; cannot read' in content:
-            debug.show('content[:64]')
-            debug.show('utext[:64]')
-            log.assertion('content == utext')
+        content = doc.text_or_error()     # pyflakes:ignore
         content = markup_txt.markup(content)
 
         ballot_summary = None
@@ -502,12 +479,7 @@ def document_main(request, name, rev=None):
             # This could move to a template
             content = u"A conflict review response has not yet been proposed."
         else:     
-            content = get_document_content(filename, pathname, split=False, markup=True)
-            utext = doc.text_or_error() # pyflakes:ignore
-            if content and content != utext and not 'Error; cannot read' in content:
-                debug.show('content[:64]')
-                debug.show('utext[:64]')
-                log.assertion('content == utext')
+            content = doc.text_or_error() # pyflakes:ignore
             content = markup_txt.markup(content)
 
         ballot_summary = None
@@ -535,12 +507,7 @@ def document_main(request, name, rev=None):
             # This could move to a template
             content = u"Status change text has not yet been proposed."
         else:     
-            content = get_document_content(filename, pathname, split=False)
-            utext = doc.text_or_error() # pyflakes:ignore
-            if content and content != utext and not 'Error; cannot read' in content:
-                debug.show('content[:64]')
-                debug.show('utext[:64]')
-                log.assertion('content == utext')
+            content = doc.text_or_error() # pyflakes:ignore
 
         ballot_summary = None
         if doc.get_state_slug() in ("iesgeval"):
@@ -593,12 +560,7 @@ def document_main(request, name, rev=None):
                 url = urlbase + extension 
 
             if extension == ".txt":
-                content = get_document_content(basename, pathname + extension, split=False)
-                utext = doc.text_or_error()      # pyflakes:ignore
-                if content != utext:
-                    debug.show('content[:64]')
-                    debug.show('utext[:64]')
-                log.assertion('content == utext')
+                content = doc.text_or_error()      # pyflakes:ignore
                 t = "plain text"
 
             other_types.append((t, url))

ietf/doc/views_status_change.py

@@ -13,7 +13,7 @@
     BallotPositionDocEvent, NewRevisionDocEvent, WriteupDocEvent, STATUSCHANGE_RELATIONS )
 from ietf.doc.forms import AdForm
 from ietf.doc.lastcall import request_last_call
-from ietf.doc.utils import get_document_content, add_state_change_event, update_telechat, close_open_ballots, create_ballot_if_not_open
+from ietf.doc.utils import add_state_change_event, update_telechat, close_open_ballots, create_ballot_if_not_open
 from ietf.doc.views_ballot import LastCallTextForm
 from ietf.group.models import Group
 from ietf.iesg.models import TelechatDate
@@ -281,13 +281,7 @@ def newstatus(relateddoc):
 
 def default_approval_text(status_change,relateddoc):
 
-    filename = "%s-%s.txt" % (status_change.canonical_name(), status_change.rev)
-    current_text = get_document_content(filename, os.path.join(settings.STATUS_CHANGE_PATH, filename), split=False, markup=False)
-    utext = status_change.text_or_error() # pyflakes:ignore
-    if current_text and current_text != utext and not 'Error; cannot read' in current_text:
-        debug.show('current_text[:64]')
-        debug.show('utext[:64]')
-        log.assertion('current_text == utext')
+    current_text = status_change.text_or_error() # pyflakes:ignore
 
     if relateddoc.target.document.std_level.slug in ('std','ps','ds','bcp',):
         action = "Protocol Action"

ietf/meeting/forms.py

@@ -9,7 +9,6 @@
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import Document, DocAlias, State, NewRevisionDocEvent
-from ietf.doc.utils import get_document_content
 from ietf.group.models import Group
 from ietf.ietfauth.utils import has_role
 from ietf.meeting.models import Session, Meeting, Schedule, countries, timezones
@@ -18,7 +17,6 @@
 from ietf.message.models import Message
 from ietf.person.models import Person
 from ietf.utils.fields import DatepickerDateField, DurationField
-from ietf.utils import log
 
 # need to insert empty option for use in ChoiceField
 # countries.insert(0, ('', '-'*9 ))
@@ -220,13 +218,7 @@ def __init__(self, *args, **kwargs):
             self.initial['time'] = self.instance.official_timeslotassignment().timeslot.time
             if self.instance.agenda():
                 doc = self.instance.agenda()
-                path = os.path.join(doc.get_file_path(), doc.filename_with_rev())
-                content = get_document_content(os.path.basename(path), path, markup=False)
-                utext = doc.text_or_error() # pyflakes:ignore
-                if content and content != utext and not 'Error; cannot read' in content:
-                    debug.show('content[:64]')
-                    debug.show('utext[:64]')
-                    log.assertion('content == utext')
+                content = doc.text_or_error()
                 self.initial['agenda'] = content
 
 

ietf/secr/telechat/views.py

@@ -1,4 +1,3 @@
-import os
 import datetime
 
 from django.contrib import messages
@@ -9,16 +8,14 @@
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import DocEvent, Document, BallotDocEvent, BallotPositionDocEvent, BallotType, WriteupDocEvent
-from ietf.doc.utils import get_document_content, add_state_change_event
+from ietf.doc.utils import add_state_change_event
 from ietf.person.models import Person
 from ietf.doc.lastcall import request_last_call
 from ietf.doc.mails import email_state_changed
 from ietf.iesg.models import TelechatDate, TelechatAgendaItem, Telechat
 from ietf.iesg.agenda import agenda_data, get_doc_section
 from ietf.ietfauth.utils import role_required
 from ietf.secr.telechat.forms import BallotForm, ChangeStateForm, DateSelectForm, TELECHAT_TAGS
-from ietf.utils import log
-
 
 '''
 EXPECTED CHANGES:
@@ -71,13 +68,7 @@ def get_doc_writeup(doc):
         if latest:
             writeup = latest.text
     elif doc.type_id == 'conflrev':
-        path = os.path.join(doc.get_file_path(),doc.filename_with_rev())
-        writeup = get_document_content(doc.name,path,split=False,markup=False)
-        utext = doc.text_or_error()     # pyflakes:ignore
-        if writeup and writeup != utext and not 'Error; cannot read' in writeup:
-            debug.show('writeup[:64]')
-            debug.show('utext[:64]')
-            log.assertion('writeup == utext')
+        writeup = doc.text_or_error()     # pyflakes:ignore
     return writeup
 
 def get_last_telechat_date():