ietf-tools · rjsparks · Jan 11, 2024 · Jan 11, 2024
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
@@ -51,8 +51,6 @@
 from django import forms
 from django.contrib.staticfiles import finders
 
-import rfc2html
-
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import ( Document, DocHistory, DocEvent, BallotDocEvent, BallotType,
@@ -82,7 +80,7 @@
 from ietf.review.models import ReviewAssignment
 from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs, review_requests_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
-from ietf.utils import log, markdown
+from ietf.utils import markup_txt, log, markdown
 from ietf.utils.draft import PlaintextDraft
 from ietf.utils.response import permission_denied
 from ietf.utils.text import maybe_split
@@ -266,9 +264,7 @@ def document_main(request, name, rev=None, document_html=False):
 
         file_urls, found_types = build_file_urls(doc)
         content = doc.text_or_error() # pyflakes:ignore
-        content = maybe_split(content, split=split_content)
-        if content:
-            content = rfc2html.markup(content)
+        content = markup_txt.markup(maybe_split(content, split=split_content))
 
         if not found_types:
             content = "This RFC is not currently available online."
@@ -404,9 +400,7 @@ def document_main(request, name, rev=None, document_html=False):
 
         file_urls, found_types = build_file_urls(doc)
         content = doc.text_or_error() # pyflakes:ignore
-        content = maybe_split(content, split=split_content)
-        if content:
-            content = rfc2html.markup(content)
+        content = markup_txt.markup(maybe_split(content, split=split_content))
 
         latest_revision = doc.latest_event(NewRevisionDocEvent, type="new_revision")
 
@@ -781,7 +775,7 @@ def document_main(request, name, rev=None, document_html=False):
             content = "A conflict review response has not yet been proposed."
         else:     
             content = doc.text_or_error() # pyflakes:ignore
-            content = markdown.markdown(content)
+            content = markup_txt.markup(content)
 
         ballot_summary = None
         if doc.get_state_slug() in ("iesgeval", ) and doc.active_ballot():
@@ -896,6 +890,8 @@ def document_main(request, name, rev=None, document_html=False):
         basename = "{}.txt".format(doc.name)
         pathname = os.path.join(doc.get_file_path(), basename)
         content = get_unicode_document_content(basename, pathname)
+        # If we want to go back to using markup_txt.markup_unicode, call it explicitly here like this:
+        # content = markup_txt.markup_unicode(content, split=False, width=80)
 
         assignments = ReviewAssignment.objects.filter(review__name=doc.name)
         review_assignment = assignments.first()

diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html
@@ -15,15 +15,7 @@
     <meta name="description"
           content="{{ doc.title }} {% if doc.type_id == 'rfc' and not snapshot %}(RFC {{ rfc_number }}{% if published %}, {{ doc.pub_date|date:'F Y' }}{% endif %}{% if obsoleted_by %}; obsoleted by {% for rel in obsoleted_by %}{{ rel.source.name|prettystdname}}{% if not forloop.last%}, {% endif %}{% endfor %}{% endif %}){% endif %}">
 {% endblock %}
-{% block morecss %}
-    .rfcmarkup :is(h1, h2, h3, h4, h5, h6),
-    .rfcmarkup :is(.h1, .h2, .h3, .h4, .h5, .h6) {
-        font-weight: bold;
-        font-size: 1em;
-    }
-    .rfcmarkup .newpage { margin-top: -1.5em; }
-    .rfcmarkup .grey, hr { opacity: .25; }
-{% endblock %}
+{% block morecss %}.inline { display: inline; }{% endblock %}
 {% block title %}
     {% if doc.type_id == "rfc" and not snapshot %}
         RFC {{ rfc_number }} - {{ doc.title }}
@@ -666,7 +658,7 @@
                 </i>
                 Nits
             </a>
-            <div class="dropdown d-inline">
+            <div class="dropdown inline">
                 <button class="btn btn-primary btn-sm dropdown-toggle"
                         type="button"
                         id="ddSearchMenu"
@@ -768,9 +760,8 @@
                         {{ name }}-{{ doc.rev }}
                     {% endif %}
                 </div>
-                <div class="card-body rfcmarkup">
-                    <!-- [html-validate-disable-block attr-quotes, void-style, element-permitted-content, heading-level, no-dup-id, valid-id -- FIXME: rfcmarkup/rfc2html generates HTML with issues] -->
-                    {{ content|safe|default:"(Unavailable)" }}
+                <div class="card-body">
+                    <pre>{{ content|sanitize|safe|default:"(Unavailable)" }}</pre>
                 </div>
             </div>
             {% if split_content %}

diff --git a/ietf/templates/doc/document_html.html b/ietf/templates/doc/document_html.html
@@ -165,7 +165,7 @@
                 {% else %}
                     <div class="rfcmarkup">
                         <br class="noprint">
-                        <!-- [html-validate-disable-block attr-quotes, void-style, element-permitted-content, heading-level, no-dup-id, valid-id -- FIXME: rfcmarkup/rfc2html generates HTML with issues] -->
+                        <!-- [html-validate-disable-block attr-quotes, void-style, element-permitted-content, heading-level -- FIXME: rfcmarkup/rfc2html generates HTML with issues] -->
                         {{ doc.htmlized|default:"Generation of htmlized text failed"|safe }}
                     </div>
                 {% endif %}
@@ -332,4 +332,4 @@
         </div>
     {% analytical_body_bottom %}
     </body>
-</html>
+</html>
diff --git a/ietf/templates/doc/document_rfc.html b/ietf/templates/doc/document_rfc.html
@@ -15,15 +15,7 @@
     <meta name="description"
           content="{{ doc.title }} (RFC {{ doc.rfc_number }}, {{ doc.pub_date|date:'F Y' }}{% if obsoleted_by %}; obsoleted by {% for rel in obsoleted_by %}{{ rel.source.name|prettystdname}}{% if not forloop.last%}, {% endif %}{% endfor %}{% endif %})">
 {% endblock %}
-{% block morecss %}
-    .rfcmarkup :is(h1, h2, h3, h4, h5, h6),
-    .rfcmarkup :is(.h1, .h2, .h3, .h4, .h5, .h6) {
-        font-weight: bold;
-        font-size: 1em;
-    }
-    .rfcmarkup .newpage { margin-top: -1.5em; }
-    .rfcmarkup .grey, hr { opacity: .25; }
-{% endblock %}
+{% block morecss %}.inline { display: inline; }{% endblock %}
 {% block title %}
         RFC {{ doc.rfc_number }} - {{ doc.title }}
 {% endblock %}
@@ -163,9 +155,8 @@
             <div class="card-header">
                     RFC {{ doc.rfc_number }}
             </div>
-                <div class="card-body rfcmarkup">
-                    <!-- [html-validate-disable-block attr-quotes, void-style, element-permitted-content, heading-level, no-dup-id, valid-id -- FIXME: rfcmarkup/rfc2html generates HTML with issues] -->
-                    {{ content|safe|default:"(Unavailable)" }}
+            <div class="card-body">
+                <pre>{{ content|sanitize|safe|default:"(Unavailable)" }}</pre>
             </div>
         </div>
         {% if split_content %}
@@ -182,4 +173,4 @@
         </script>
         <script src="{% static 'ietf/js/document_timeline.js' %}">
         </script>
-    {% endblock %}
+    {% endblock %}
diff --git a/ietf/utils/markup_txt.py b/ietf/utils/markup_txt.py
@@ -0,0 +1,69 @@
+# Copyright The IETF Trust 2009-2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above
+#    copyright notice, this list of conditions and the following
+#    disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#
+#  * Neither the name of the Nokia Corporation and/or its
+#    subsidiary(-ies) nor the names of its contributors may be used
+#    to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import re
+
+from django.utils.html import escape
+
+from ietf.utils import log
+from ietf.utils.text import wordwrap
+
+def markup(content, width=None):
+    log.assertion('isinstance(content, str)')
+    # normalize line endings to LF only
+    content = content.replace("\r\n", "\n")
+    content = content.replace("\r", "\n")
+
+    # remove leading white space
+    content = content.lstrip()
+    # remove runs of blank lines
+    content = re.sub("\n\n\n+", "\n\n", content)
+
+    # maybe wordwrap.  This must be done before the escaping below.
+    if width:
+        content = wordwrap(content, width)
+
+    # expand tabs + escape 
+    content = escape(content.expandtabs())
+
+    content = re.sub(r"\n(.+\[Page \d+\])\n\f\n(.+)\n", r"""\n<span class="m_ftr">\g<1></span>\n<span class="m_hdr">\g<2></span>\n""", content)
+    content = re.sub(r"\n(.+\[Page \d+\])\n\s*$", r"""\n<span class="m_ftr">\g<1></span>\n""", content)
+    # remove remaining FFs (to be valid XHTML)
+    content = content.replace("\f","\n")
+
+    content = re.sub(r"\n\n([0-9]+\\.|[A-Z]\\.[0-9]|Appendix|Status of|Abstract|Table of|Full Copyright|Copyright|Intellectual Property|Acknowled|Author|Index)(.*)(?=\n\n)", r"""\n\n<span class="m_h">\g<1>\g<2></span>""", content)
+
+    return content
diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py
@@ -196,7 +196,6 @@ def vnu_filter_message(msg, filter_db_issues, filter_test_issues):
     if filter_test_issues and re.search(
         r"""Ceci\ n'est\ pas\ une\ URL|
             ^The\ '\w+'\ attribute\ on\ the\ '\w+'\ element\ is\ obsolete|
-            ^Duplicate\ ID|^The\ first\ occurrence\ of\ ID|
             ^Section\ lacks\ heading""",
         msg["message"],
         flags=re.VERBOSE,