feat: better htmlization views

dev/tests/debug.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# This script recreate the same environment used during tests on GitHub Actions
+# and drops you into a terminal at the point where the actual tests would be run.
+#
+# Refer to https://github.com/ietf-tools/datatracker/blob/main/.github/workflows/build.yml#L141-L155
+# for the commands to run next.
+#
+# Simply type "exit" + ENTER to exit and shutdown this test environment.
+
+echo "Fetching latest images..."
+docker pull ghcr.io/ietf-tools/datatracker-app-base:latest
+docker pull ghcr.io/ietf-tools/datatracker-db:latest
+echo "Starting containers..."
+docker compose -f docker-compose.debug.yml -p dtdebug up -d
+echo "Copying working directory into container..."
+docker compose -p dtdebug cp ../../. app:/__w/datatracker/datatracker/
+echo "Run prepare script..."
+docker compose -p dtdebug exec app chmod +x ./dev/tests/prepare.sh
+docker compose -p dtdebug exec app sh ./dev/tests/prepare.sh
+docker compose -p dtdebug exec app /usr/local/bin/wait-for db:3306 -- echo "DB ready"
+echo "================================================================="
+echo "Launching zsh terminal:"
+docker compose -p dtdebug exec app /bin/zsh
+echo "Shutting down containers..."
+docker compose -p dtdebug down -v

dev/tests/docker-compose.debug.yml

@@ -0,0 +1,33 @@
+# This docker-compose replicates the test workflow happening on GitHub during a PR / build check.
+# To be used from the debug.sh script.
+
+version: '3.8'
+
+services:
+    app:
+        image: ghcr.io/ietf-tools/datatracker-app-base:latest
+        command: -f /dev/null
+        working_dir: /__w/datatracker/datatracker
+        entrypoint: tail
+        hostname: app
+        volumes:
+            - /var/run/docker.sock:/var/run/docker.sock
+        environment:
+            CI: 'true'
+            GITHUB_ACTIONS: 'true'
+            HOME: /github/home
+    db:
+        image: ghcr.io/ietf-tools/datatracker-db:latest
+        restart: unless-stopped
+        volumes:
+            - mariadb-data:/var/lib/mysql
+        environment:
+            MYSQL_ROOT_PASSWORD: RkTkDPFnKpko
+            MYSQL_DATABASE: ietf_utf8
+            MYSQL_USER: django
+            MYSQL_PASSWORD: RkTkDPFnKpko
+            CI: 'true'
+            GITHUB_ACTIONS: 'true'
+
+volumes:
+    mariadb-data:

ietf/doc/models.py

@@ -8,6 +8,8 @@
 import os
 import rfc2html
 
+from pathlib import Path
+from lxml import etree
 from typing import Optional, TYPE_CHECKING
 from weasyprint import HTML as wpHTML
 
@@ -21,6 +23,7 @@
 from django.utils import timezone
 from django.utils.encoding import force_text
 from django.utils.html import mark_safe # type:ignore
+from django.contrib.staticfiles import finders
 
 import debug                            # pyflakes:ignore
 
@@ -541,6 +544,46 @@ def text(self):
     def text_or_error(self):
         return self.text() or "Error; cannot read '%s'"%self.get_base_name()
 
+    def html_body(self, classes=""):
+        if self.get_state_slug() == "rfc":
+            try:
+                html = Path(
+                    os.path.join(settings.RFC_PATH, self.canonical_name() + ".html")
+                ).read_text()
+            except IOError:
+                return None
+        else:
+            try:
+                html = Path(
+                    os.path.join(
+                        settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR,
+                        self.name + "-" + self.rev + ".html",
+                    )
+                ).read_text()
+            except IOError:
+                return None
+
+        # If HTML was generated by rfc2html, do not return it. Caller
+        # will use htmlize() to use a more current rfc2html to
+        # generate an HTMLized version. TODO: There should be a
+        # better way to determine how an HTML format was generated.
+        if html.startswith("<pre>"):
+            return None
+
+        # get body
+        body = etree.HTML(html).xpath("//body")[0]
+        body.tag = "div"
+        if classes:
+            body.attrib["class"] = classes
+
+        # remove things
+        for tag in ["script"]:
+            for t in body.xpath(f"//{tag}"):
+                t.getparent().remove(t)
+        html = etree.tostring(body, encoding=str, method="html")
+
+        return html
+
     def htmlized(self):
         name = self.get_base_name()
         text = self.text()
@@ -566,17 +609,29 @@ def htmlized(self):
 
     def pdfized(self):
         name = self.get_base_name()
-        text = self.text()
-        cache = caches['pdfized']
-        cache_key = name.split('.')[0]
+        text = self.html_body(classes="rfchtml")
+        stylesheets = [finders.find("ietf/css/document_html_referenced.css")]
+        if text:
+            stylesheets.append(finders.find("ietf/css/document_html_txt.css"))
+        else:
+            text = self.htmlized()
+            stylesheets.append(io.BytesIO(b"body { font-size: 9.2pt; }"))
+
+        cache = caches["pdfized"]
+        cache_key = name.split(".")[0]
         try:
             pdf = cache.get(cache_key)
         except EOFError:
             pdf = None
         if not pdf:
-            html = rfc2html.markup(text, path=settings.PDFIZER_URL_PREFIX)
             try:
-                pdf = wpHTML(string=html.replace('\xad','')).write_pdf(stylesheets=[io.BytesIO(b'html { font-size: 94%;}')])
+                pdf = wpHTML(
+                    string=text, base_url=settings.IDTRACKER_BASE_URL
+                ).write_pdf(
+                    stylesheets=stylesheets,
+                    presentational_hints=True,
+                    optimize_size=("fonts", "images"),
+                )
             except AssertionError:
                 pdf = None
             if pdf:

ietf/doc/templatetags/ietf_filters.py

@@ -260,8 +260,8 @@ def urlize_ietf_docs(string, autoescape=None):
 
 urlize_ietf_docs = stringfilter(urlize_ietf_docs)
 
-@register.filter(name='urlize_related_source_list', is_safe=True, needs_autoescape=True)
-def urlize_related_source_list(related, autoescape=None):
+@register.filter(name='urlize_related_source_list', is_safe=True, document_html=False)
+def urlize_related_source_list(related, document_html=False):
     """Convert a list of RelatedDocuments into list of links using the source document's canonical name"""
     links = []
     names = set()
@@ -273,28 +273,26 @@ def urlize_related_source_list(related, autoescape=None):
             continue
         names.add(name)
         titles.add(title)
-        url = urlreverse('ietf.doc.views_doc.document_main', kwargs=dict(name=name))
-        if autoescape:
-            name = escape(name)
-            title = escape(title)
+        url = urlreverse('ietf.doc.views_doc.document_main' if document_html is False else 'ietf.doc.views_doc.document_html', kwargs=dict(name=name))
+        name = escape(name)
+        title = escape(title)
         links.append(mark_safe(
             '<a href="%(url)s" title="%(title)s">%(name)s</a>' % dict(name=prettify_std_name(name),
                                                                       title=title,
                                                                       url=url)
         ))
     return links
 
-@register.filter(name='urlize_related_target_list', is_safe=True, needs_autoescape=True)
-def urlize_related_target_list(related, autoescape=None):
+@register.filter(name='urlize_related_target_list', is_safe=True, document_html=False)
+def urlize_related_target_list(related, document_html=False):
     """Convert a list of RelatedDocuments into list of links using the target document's canonical name"""
     links = []
     for rel in related:
         name=rel.target.document.canonical_name()
         title = rel.target.document.title
-        url = urlreverse('ietf.doc.views_doc.document_main', kwargs=dict(name=name))
-        if autoescape:
-            name = escape(name)
-            title = escape(title)
+        url = urlreverse('ietf.doc.views_doc.document_main' if document_html is False else 'ietf.doc.views_doc.document_html', kwargs=dict(name=name))
+        name = escape(name)
+        title = escape(title)
         links.append(mark_safe(
             '<a href="%(url)s" title="%(title)s">%(name)s</a>' % dict(name=prettify_std_name(name),
                                                                       title=title,
@@ -554,6 +552,19 @@ def consensus(doc):
     else:
         return "Unknown"
 
+
+@register.filter
+def std_level_to_label_format(doc):
+    """Returns valid Bootstrap classes to label a status level badge."""
+    if doc.is_rfc():
+        if doc.related_that("obs"):
+            return "obs"
+        else:
+            return doc.std_level_id
+    else:
+        return "draft"
+
+
 @register.filter
 def pos_to_label_format(text):
     """Returns valid Bootstrap classes to label a ballot position."""

ietf/doc/tests.py

@@ -730,13 +730,13 @@ def test_document_draft(self):
 
         r = self.client.get(urlreverse("ietf.doc.views_doc.document_html", kwargs=dict(name=draft.name)))
         self.assertEqual(r.status_code, 200)
-        self.assertContains(r, "Versions:")
+        self.assertContains(r, "Select version")
         self.assertContains(r, "Deimos street")
         q = PyQuery(r.content)
         self.assertEqual(q('title').text(), 'draft-ietf-mars-test-01')
-        self.assertEqual(len(q('.rfcmarkup pre')), 4)
-        self.assertEqual(len(q('.rfcmarkup span.h1')), 2)
-        self.assertEqual(len(q('.rfcmarkup a[href]')), 41)
+        self.assertEqual(len(q('.rfcmarkup pre')), 3)
+        self.assertEqual(len(q('.rfcmarkup span.h1, .rfcmarkup h1')), 2)
+        self.assertEqual(len(q('.rfcmarkup a[href]')), 28)
 
         r = self.client.get(urlreverse("ietf.doc.views_doc.document_html", kwargs=dict(name=draft.name, rev=draft.rev)))
         self.assertEqual(r.status_code, 200)