feat: Render the document shepherd writeup templates at two new URLs (ietf-tools#4225)

* feat: Render the document shepherd writeup templates at two new URL.

Those being `/doc/shepherdwriteuptemplate/group` and
`/doc/shepherdwriteuptemplate/individual`.

* Address review comments from @jennifer-richards

* Fixes

* Remove debug statement

* Make bleach sanitizer not strip the `start` attribute of `ol` tags

Also rearrange the code a bit

* Don't sanitize the `python_markdown` output, it destroys wanted formatting

* Restore bleach

* Don't bleach tag `id`s.

Author: larseggert

ietf/doc/tests_draft.py

@@ -1163,7 +1163,27 @@ def test_doc_change_shepherd_email(self):
         self.assertEqual(r.status_code, 302)
         doc = Document.objects.get(name=self.docname)
         self.assertEqual(comment_event, doc.latest_event(DocEvent, type="added_comment"))
-       
+
+    def test_doc_view_shepherd_writeup_templates(self):
+        url = urlreverse(
+            "ietf.doc.views_doc.document_shepherd_writeup_template",
+            kwargs=dict(type="group"),
+        )
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q('h1:contains("for Group Documents")')), 1)
+
+        url = urlreverse(
+            "ietf.doc.views_doc.document_shepherd_writeup_template",
+            kwargs=dict(type="individual"),
+        )
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q('h1:contains("for Individual Documents")')), 1)
 
     def test_doc_view_shepherd_writeup(self):
         url = urlreverse('ietf.doc.views_doc.document_shepherd_writeup',kwargs=dict(name=self.docname))

ietf/doc/urls.py

@@ -60,6 +60,11 @@
     url(r'^email-aliases/?$', views_doc.email_aliases),
     url(r'^downref/?$', views_downref.downref_registry),
     url(r'^downref/add/?$', views_downref.downref_registry_add),
+    url(
+        r"^shepherdwriteup-template/(?P<type>\w+)/?$",
+        views_doc.document_shepherd_writeup_template,
+    ),
+
     url(r'^stats/newrevisiondocevent/?$', views_stats.chart_newrevisiondocevent),
     url(r'^stats/newrevisiondocevent/conf/?$', views_stats.chart_conf_newrevisiondocevent),
     url(r'^stats/newrevisiondocevent/data/?$', views_stats.chart_data_newrevisiondocevent),

ietf/doc/views_doc.py

@@ -1139,6 +1139,25 @@ def document_shepherd_writeup(request, name):
                                    ),
                               )
 
+
+def document_shepherd_writeup_template(request, type):
+    writeup = markdown.markdown(
+        render_to_string(
+            "doc/shepherd_writeup.txt",
+            dict(stream="ietf", type="individ" if type == "individual" else "group"),
+        )
+    )
+    return render(
+        request,
+        "doc/shepherd_writeup_template.html",
+        dict(
+            writeup=writeup,
+            stream="ietf",
+            type="individ" if type == "individual" else "group",
+        ),
+    )
+
+
 def document_references(request, name):
     doc = get_object_or_404(Document,docalias__name=name)
     refs = doc.references()

ietf/doc/views_draft.py

@@ -940,49 +940,68 @@ def clean_content(self):
     def clean_txt(self):
         return get_cleaned_text_file_content(self.cleaned_data["txt"])
 
+
 @login_required
 def edit_shepherd_writeup(request, name):
     """Change this document's shepherd writeup"""
     doc = get_object_or_404(Document, type="draft", name=name)
 
     can_edit_stream_info = is_authorized_in_doc_stream(request.user, doc)
-    can_edit_shepherd_writeup = ( can_edit_stream_info
+    can_edit_shepherd_writeup = (
+        can_edit_stream_info
         or (doc.shepherd and user_is_person(request.user, doc.shepherd.person))
-        or has_role(request.user, ["Area Director"]))
+        or has_role(request.user, ["Area Director"])
+    )
 
     if not can_edit_shepherd_writeup:
-        permission_denied(request, "You do not have the necessary permissions to view this page")
+        permission_denied(
+            request, "You do not have the necessary permissions to view this page"
+        )
 
     login = request.user.person
 
-    if request.method == 'POST':
+    if request.method == "POST":
         if "submit_response" in request.POST:
             form = ShepherdWriteupUploadForm(request.POST, request.FILES)
             if form.is_valid():
-                
-                from_file = form.cleaned_data['txt']
+
+                from_file = form.cleaned_data["txt"]
                 if from_file:
-                     writeup = from_file
+                    writeup = from_file
                 else:
-                     writeup = form.cleaned_data['content']
-                e = WriteupDocEvent(doc=doc, rev=doc.rev, by=login, type="changed_protocol_writeup")
+                    writeup = form.cleaned_data["content"]
+                e = WriteupDocEvent(
+                    doc=doc, rev=doc.rev, by=login, type="changed_protocol_writeup"
+                )
 
-                # Add the shepherd writeup to description if the document is in submitted for publication state
+                # Add the shepherd writeup to description,
+                # if the document is in submitted for publication state
                 stream_state = doc.get_state("draft-stream-%s" % doc.stream_id)
-                iesg_state   = doc.get_state("draft-iesg")
-                if (iesg_state or (stream_state and stream_state.slug=='sub-pub')):
+                iesg_state = doc.get_state("draft-iesg")
+                if iesg_state or (stream_state and stream_state.slug == "sub-pub"):
                     e.desc = writeup
                 else:
                     e.desc = "Changed document writeup"
 
                 e.text = writeup
                 e.save()
-            
-                return redirect('ietf.doc.views_doc.document_main', name=doc.name)
 
-        elif "reset_text" in request.POST:
+                return redirect("ietf.doc.views_doc.document_main", name=doc.name)
 
-            init = { "content": render_to_string("doc/shepherd_writeup.txt",dict(doc=doc))}
+        elif "reset_text" in request.POST:
+            init = {
+                "content": render_to_string(
+                    "doc/shepherd_writeup.txt",
+                    dict(
+                        doc=doc,
+                        type="individ"
+                        if not doc.group.type.slug or doc.group.type.slug != "ietf"
+                        else "group",
+                        stream=doc.stream.slug,
+                        group=doc.group.type.slug,
+                    ),
+                )
+            }
             form = ShepherdWriteupUploadForm(initial=init)
 
         # Protect against handcrufted malicious posts
@@ -993,21 +1012,36 @@ def edit_shepherd_writeup(request, name):
         form = None
 
     if not form:
-        init = { "content": ""}
+        init = {"content": ""}
 
-        previous_writeup = doc.latest_event(WriteupDocEvent,type="changed_protocol_writeup")
+        previous_writeup = doc.latest_event(
+            WriteupDocEvent, type="changed_protocol_writeup"
+        )
         if previous_writeup:
             init["content"] = previous_writeup.text
         else:
-            init["content"] = render_to_string("doc/shepherd_writeup.txt",
-                                                dict(doc=doc),
-                                              )
+            init["content"] = render_to_string(
+                "doc/shepherd_writeup.txt",
+                dict(
+                    doc=doc,
+                    type="individ"
+                    if not doc.group.type.slug or doc.group.type.slug != "ietf"
+                    else "group",
+                    stream=doc.stream.slug,
+                    group=doc.group.type.slug,
+                ),
+            )
         form = ShepherdWriteupUploadForm(initial=init)
 
-    return render(request, 'doc/draft/change_shepherd_writeup.html',
-                              {'form': form,
-                               'doc' : doc,
-                              })
+    return render(
+        request,
+        "doc/draft/change_shepherd_writeup.html",
+        {
+            "form": form,
+            "doc": doc,
+        },
+    )
+
 
 class ShepherdForm(forms.Form):
     shepherd = SearchableEmailField(required=False, only_users=True)

ietf/templates/doc/shepherd_writeup.txt

@@ -1,4 +1,4 @@
-{# Keep in sync with https://github.com/ietf-chairs/chairs.ietf.org/blob/main/documents/qa-style-writeup-template.md #}{% if doc.stream %}{% if doc.stream.slug == 'ietf' %}# Document Shepherd Write-Up
+{# Keep in sync with https://github.com/ietf-chairs/chairs.ietf.org/blob/main/documents/qa-style-writeup-template.md #}{% if stream %}{% if stream == 'ietf' %}# Document Shepherd Write-Up{% if type != "individ" %} for Group Documents{% else %} for Individual Documents{% endif %}
 
 *This version is dated 4 July 2022.*
 
@@ -13,7 +13,7 @@ Note that some numbered items contain multiple related questions; please be sure
 to answer all of them.
 
 ## Document History
-{% if doc.group.type.slug == 'individ' %}
+{% if type == 'individ' %}
 1. Was the document considered in any WG, and if so, why was it not adopted as a
    work item there?
 
@@ -37,7 +37,7 @@ to answer all of them.
    either in the document itself (as [RFC 7942][3] recommends) or elsewhere
    (where)?
 
-### Additional Reviews
+## Additional Reviews
 
 5. Do the contents of this document closely interact with technologies in other
    IETF working groups or external organizations, and would it therefore benefit
@@ -58,7 +58,7 @@ to answer all of them.
    final version of the document written in a formal language, such as XML code,
    BNF rules, MIB definitions, CBOR's CDDL, etc.
 
-### Document Shepherd Checks
+## Document Shepherd Checks
 
 9. Based on the shepherd's review of the document, is it their opinion that this
    document is needed, clearly written, complete, correctly designed, and ready
@@ -139,5 +139,5 @@ to answer all of them.
 [15]: https://authors.ietf.org/en/content-guidelines-overview
 [16]: https://www.ietf.org/about/groups/iesg/statements/normative-informative-references/
 [17]: https://datatracker.ietf.org/doc/downref/
-{% else %}There is no default shepherd write-up template for the {{doc.stream}} stream.
+{% else %}There is no default shepherd write-up template for the {{stream}} stream.
 {% endif %}{% else %}There is no stream set for this document (thus, no default shepherd write-up template).{% endif %}

ietf/templates/doc/shepherd_writeup_template.html

@@ -0,0 +1,10 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{% load origin %}
+{% load ietf_filters %}
+{% load textfilters htmlfilters %}
+{% block title %}Document Shepherd Write-Up{% if type == "group" %} for Group Documents{% elif type == "individual" %} for Individual Documents{% endif %}{% endblock %}
+{% block content %}
+    {% origin %}
+    {{ writeup|urlize_ietf_docs|linkify }}
+{% endblock %}

ietf/utils/markdown.py

@@ -8,7 +8,6 @@
 import markdown as python_markdown
 
 from django.utils.safestring import mark_safe
-from markdown.extensions.extra import ExtraExtension
 
 from ietf.doc.templatetags.ietf_filters import urlize_ietf_docs
 from ietf.utils.text import bleach_cleaner, bleach_linker
@@ -20,7 +19,7 @@ def markdown(text):
             urlize_ietf_docs(
                 bleach_cleaner.clean(
                     python_markdown.markdown(
-                        text, extensions=[ExtraExtension(), "nl2br"]
+                        text, extensions=["extra", "nl2br", "sane_lists", "toc"]
                     )
                 )
             )

ietf/utils/text.py

@@ -23,13 +23,36 @@
 protocols = copy.copy(bleach.sanitizer.ALLOWED_PROTOCOLS)
 protocols.append("ftp")  # we still have some ftp links
 protocols.append("xmpp")  # we still have some xmpp links
+
+tags = set(copy.copy(bleach.sanitizer.ALLOWED_TAGS)).union(
+    {
+        # fmt: off
+        'a', 'abbr', 'acronym', 'address', 'b', 'big',
+        'blockquote', 'body', 'br', 'caption', 'center', 'cite', 'code', 'col',
+        'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt', 'em', 'font',
+        'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'head', 'hr', 'html', 'i', 'ins', 'kbd',
+        'li', 'ol', 'p', 'pre', 'q', 's', 'samp', 'small', 'span', 'strike', 'style',
+        'strong', 'sub', 'sup', 'table', 'title', 'tbody', 'td', 'tfoot', 'th', 'thead',
+        'tr', 'tt', 'u', 'ul', 'var'
+        # fmt: on
+    }
+)
+
+attributes = copy.copy(bleach.sanitizer.ALLOWED_ATTRIBUTES)
+attributes["*"] = ["id"]
+attributes["ol"] = ["start"]
+
+bleach_cleaner = bleach.sanitizer.Cleaner(
+    tags=tags, attributes=attributes, protocols=protocols, strip=True
+)
+
 validate_url = URLValidator()
 
 
 def check_url_validity(attrs, new=False):
-    if (None, 'href') not in attrs:
+    if (None, "href") not in attrs:
         return None
-    url = attrs[(None, 'href')]
+    url = attrs[(None, "href")]
     try:
         if url.startswith("http"):
             validate_url(url)
@@ -41,22 +64,10 @@ def check_url_validity(attrs, new=False):
 bleach_linker = bleach.Linker(
     callbacks=[check_url_validity],
     url_re=bleach.linkifier.build_url_re(tlds=tlds_sorted, protocols=protocols),
-    email_re=bleach.linkifier.build_email_re(tlds=tlds_sorted), # type: ignore
-    parse_email=True
-)
-
-tags = (
-    'a', 'abbr', 'acronym', 'address', 'b', 'big',
-    'blockquote', 'body', 'br', 'caption', 'center', 'cite', 'code', 'col',
-    'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt', 'em', 'font',
-    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'head', 'hr', 'html', 'i', 'ins', 'kbd',
-    'li', 'ol', 'p', 'pre', 'q', 's', 'samp', 'small', 'span', 'strike', 'style',
-    'strong', 'sub', 'sup', 'table', 'title', 'tbody', 'td', 'tfoot', 'th', 'thead',
-    'tr', 'tt', 'u', 'ul', 'var'
+    email_re=bleach.linkifier.build_email_re(tlds=tlds_sorted),  # type: ignore
+    parse_email=True,
 )
 
-bleach_cleaner = bleach.sanitizer.Cleaner(tags=tags, protocols=protocols, strip=True)
-
 
 @keep_lazy(str)
 def xslugify(value):