From e6877831e6920ec68ad68ba37918ed76c9f87aef Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 29 Oct 2024 10:05:08 -0500 Subject: [PATCH 1/3] fix: quicker calculation of status from draft text --- ietf/doc/views_doc.py | 7 ++----- ietf/utils/draft.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 915dcebde50..df6a4cbd1fd 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -84,7 +84,7 @@ from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs, review_requests_to_list_for_docs from ietf.review.utils import no_review_from_teams_on_doc from ietf.utils import markup_txt, log, markdown -from ietf.utils.draft import PlaintextDraft +from ietf.utils.draft import PlaintextDraft, get_status_from_draft_text from ietf.utils.meetecho import MeetechoAPIError, SlidesManager from ietf.utils.response import permission_denied from ietf.utils.text import maybe_split @@ -2263,10 +2263,7 @@ def idnits2_state(request, name, rev=None): else: text = doc.text() if text: - parsed_draft = PlaintextDraft( - text=doc.text(), source=name, name_from_source=False - ) - doc.deststatus = parsed_draft.get_status() + doc.deststatus = get_status_from_draft_text(text) else: doc.deststatus = "Unknown" return render( diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py index a1e79760ea5..50add5abbab 100755 --- a/ietf/utils/draft.py +++ b/ietf/utils/draft.py @@ -131,6 +131,24 @@ def acronym_match(s, l): #_debug(" s:%s; l:%s => %s; %s" % (s, l, acronym, s==acronym)) return s == acronym +def get_status_from_draft_text(text): + + # Take prefix to shortcut work over very large drafts + # 5000 is conservatively much more than a full page of characters and we + # only want the first 10 lines. + text = text.strip()[:5000] # Take prefix to shortcut work over very large drafts + text = re.sub(".\x08", "", text) # Get rid of inkribbon backspace-emphasis + text = text.replace("\r\n", "\n") # Convert DOS to unix + text = text.replace("\r", "\n") # Convert MAC to unix + lines = text.split("\n")[:10] + status = None + for line in lines: + status_match = re.search(r"^\s*Intended [Ss]tatus:\s*(.*?) ", line) + if status_match: + status = status_match.group(1) + break + return status + class Draft: """Base class for drafts From 414c931f474ed20f24a028135f41d439bb5aac35 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 29 Oct 2024 10:07:41 -0500 Subject: [PATCH 2/3] chore: remove unused import --- ietf/doc/views_doc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index df6a4cbd1fd..fb9e75b5158 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -84,7 +84,7 @@ from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs, review_requests_to_list_for_docs from ietf.review.utils import no_review_from_teams_on_doc from ietf.utils import markup_txt, log, markdown -from ietf.utils.draft import PlaintextDraft, get_status_from_draft_text +from ietf.utils.draft import get_status_from_draft_text from ietf.utils.meetecho import MeetechoAPIError, SlidesManager from ietf.utils.response import permission_denied from ietf.utils.text import maybe_split From d07cb6f239d06ef39f4dd47e04ce5da0b0b0b822 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 29 Oct 2024 10:57:49 -0500 Subject: [PATCH 3/3] fix: only read a small prefix of draft text when needed --- ietf/doc/models.py | 15 +++++++++++---- ietf/doc/views_doc.py | 4 +++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 639e6ca857a..077502db114 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -530,7 +530,7 @@ def replaces(self): def replaced_by(self): return set([ r.document for r in self.related_that("replaces") ]) - def text(self): + def text(self, size = -1): path = self.get_file_name() root, ext = os.path.splitext(path) txtpath = root+'.txt' @@ -538,14 +538,21 @@ def text(self): path = txtpath try: with io.open(path, 'rb') as file: - raw = file.read() + raw = file.read(size) except IOError: return None + text = None try: text = raw.decode('utf-8') except UnicodeDecodeError: - text = raw.decode('latin-1') - # + for back in range(1,4): + try: + text = raw[:-back].decode('utf-8') + break + except UnicodeDecodeError: + pass + if text is None: + text = raw.decode('latin-1') return text def text_or_error(self): diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index fb9e75b5158..50c60aefc59 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -2261,7 +2261,9 @@ def idnits2_state(request, name, rev=None): elif doc.intended_std_level: doc.deststatus = doc.intended_std_level.name else: - text = doc.text() + # 10000 is a conservative prefix on number of utf-8 encoded bytes to + # cover at least the first 10 lines of characters + text = doc.text(size=10000) if text: doc.deststatus = get_status_from_draft_text(text) else: