More HTML nitfixing

bin/test-crawl

@@ -44,7 +44,6 @@ args = parser.parse_args()
 
 # Import Django, call setup()
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.settings or "ietf.settings_testcrawl")
-os.environ["DJANGO_URLIZE_IETF_DOCS_PRODUCTION"] = "1"
 
 import django
 import django.test
@@ -175,7 +174,7 @@ def check_html_valid(url, response, args):
             assert ret
             for m in json.loads(ret)["messages"]:
                 if "lastLine" not in m:
-                    tag = m  # just dump the raw JSON for now
+                    tag = m["message"]
                 else:
                     tag = vnu_fmt_message(url, m, content.decode())
                 # disregard some HTML issues that are (usually) due to invalid
@@ -211,7 +210,7 @@ def skip_url(url):
             r"^/wg/[a-z0-9-]+/deps/svg/",
             # Skip other bad urls
             r"^/dir/tsvdir/reviews/",
-            r"^/ipr/\d{,3}/history/",
+            # r"^/ipr/\d{,3}/history/",
             # Skip most html conversions, not worth the time
             r"^/doc/html/draft-[0-9ac-z]",
             r"^/doc/html/draft-b[0-9b-z]",

ietf/doc/templatetags/ietf_filters.py

@@ -4,7 +4,6 @@
 
 import datetime
 import re
-import os
 from urllib.parse import urljoin
 
 from email.utils import parseaddr
@@ -19,7 +18,6 @@
 from django.utils.encoding import force_str # pyflakes:ignore force_str is used in the doctests
 from django.urls import reverse as urlreverse
 from django.core.cache import cache
-from django.core.validators import URLValidator
 from django.core.exceptions import ValidationError
 
 import debug                            # pyflakes:ignore
@@ -29,7 +27,7 @@
 from ietf.utils.html import sanitize_fragment
 from ietf.utils import log
 from ietf.doc.utils import prettify_std_name
-from ietf.utils.text import wordwrap, fill, wrap_text_if_unwrapped, bleach_linker
+from ietf.utils.text import wordwrap, fill, wrap_text_if_unwrapped, bleach_linker, bleach_cleaner, validate_url
 
 register = template.Library()
 
@@ -189,69 +187,82 @@ def rfceditor_info_url(rfcnum : str):
     return urljoin(settings.RFC_EDITOR_INFO_BASE_URL, f'rfc{rfcnum}')
 
 
-def doc_exists(name):
-    """Check whether a given document exists"""
+def doc_canonical_name(name):
+    """Check whether a given document exists, and return its canonical name"""
+
     def find_unique(n):
         key = hash(n)
         found = cache.get(key)
         if not found:
             exact = DocAlias.objects.filter(name=n).first()
             found = exact.name if exact else "_"
-            cache.set(key, found)
+            cache.set(key, found, timeout=60*60*24)  # cache for one day
         return None if found == "_" else found
 
-    # all documents exist when tests are running
-    if settings.SERVER_MODE == 'test':
-        # unless we are running test-crawl, which would otherwise 404
-        if "DJANGO_URLIZE_IETF_DOCS_PRODUCTION" not in os.environ:
-            return True
-
     # chop away extension
-    extension_split = re.search(r"^(.+)\.(txt|ps|pdf)$", name)
+    extension_split = re.search(r"^(.+)\.(txt|ps|pdf|html)$", name)
     if extension_split:
         name = extension_split.group(1)
 
     if find_unique(name):
-        return True
+        return name
 
     # check for embedded rev - this may be ambiguous, so don't
     # chop it off if we don't find a match
-    rev_split = re.search("^(.+)-([0-9]{2,})$", name)
+    rev_split = re.search(r"^(charter-.+)-(\d{2}-\d{2})$", name) or re.search(
+        r"^(.+)-(\d{2}|[1-9]\d{2,})$", name
+    )
     if rev_split:
         name = rev_split.group(1)
         if find_unique(name):
-            return True
+            return name
 
-    return False
+    return ""
 
 
 def link_charter_doc_match1(match):
-    if not doc_exists(match[0]):
+    if not doc_canonical_name(match[0]):
         return match[0]
     return f'<a href="/doc/{match[1][:-1]}/{match[2]}/">{match[0]}</a>'
 
 
 def link_charter_doc_match2(match):
-    if not doc_exists(match[0]):
+    if not doc_canonical_name(match[0]):
         return match[0]
     return f'<a href="/doc/{match[1][:-1]}/{match[2]}/">{match[0]}</a>'
 
 
 def link_non_charter_doc_match(match):
-    if not doc_exists(match[0]):
+    name = match[0]
+    cname = doc_canonical_name(name)
+    if not cname:
         return match[0]
-    if len(match[3]) == 2 and match[3].isdigit():
-        return f'<a href="/doc/{match[2][:-1]}/{match[3]}/">{match[0]}</a>'
+    if name == cname:
+        return f'<a href="/doc/{cname}/">{match[0]}</a>'
+
+    # if we get here, the name probably has a version number and/or extension at the end
+    rev_split = re.search(r"^(" + re.escape(cname) + r")-(\d{2,})", name)
+    if rev_split:
+        name = rev_split.group(1)
     else:
-        return f'<a href="/doc/{match[2]}{match[3]}/">{match[0]}</a>'
+        return f'<a href="/doc/{cname}/">{match[0]}</a>'
+
+    cname = doc_canonical_name(name)
+    if not cname:
+        return match[0]
+    if name == cname:
+        return f'<a href="/doc/{cname}/{rev_split.group(2)}/">{match[0]}</a>'
+
+    # if we get here, we can't linkify
+    return match[0]
 
 
 def link_other_doc_match(match):
-    # there may be whitespace in the match
-    doc = re.sub(r"\s+", "", match[0])
-    if not doc_exists(doc):
+    doc = match[2].strip().lower()
+    rev = match[3]
+    if not doc_canonical_name(doc + rev):
         return match[0]
-    return f'<a href="/doc/{match[2].strip().lower()}{match[3]}/">{match[1]}</a>'
+    return f'<a href="/doc/{doc}{rev}/">{match[1]}</a>'
 
 
 @register.filter(name="urlize_ietf_docs", is_safe=True, needs_autoescape=True)
@@ -264,8 +275,8 @@ def urlize_ietf_docs(string, autoescape=None):
             string = escape(string)
         else:
             string = mark_safe(string)
-    exp1 = r"\b(?<![/\-:=#])(charter-(?:[\d\w\.+]+-)*)(\d\d-\d\d)(\.txt)?\b"
-    exp2 = r"\b(?<![/\-:=#])(charter-(?:[\d\w\.+]+-)*)(\d\d)(\.txt)?\b"
+    exp1 = r"\b(?<![/\-:=#])(charter-(?:[\d\w\.+]+-)*)(\d{2}-\d{2})(\.(?:txt|ps|pdf|html))?\b"
+    exp2 = r"\b(?<![/\-:=#])(charter-(?:[\d\w\.+]+-)*)(\d{2})(\.(?:txt|ps|pdf|html))?\b"
     if re.search(exp1, string):
         string = re.sub(
             exp1,
@@ -281,7 +292,8 @@ def urlize_ietf_docs(string, autoescape=None):
             flags=re.IGNORECASE | re.ASCII,
         )
     string = re.sub(
-        r"\b(?<![/\-:=#])(((?:draft-|bofreq-|conflict-review-|status-change-)(?:[\d\w\.+]+-)*)([\d\w\.+]+?)(\.txt)?)\b(?![-@])",
+        r"\b(?<![/\-:=#])((?:draft-|bofreq-|conflict-review-|status-change-)[\d\w\.+-]+(?![-@]))",
+        # r"\b(?<![/\-:=#])(((?:draft-|bofreq-|conflict-review-|status-change-)(?:[\d\w\.+]+-)*)([\d\w\.+]+?)(\.(?:txt|ps|pdf|html))?)\b(?![-@])",
         link_non_charter_doc_match,
         string,
         flags=re.IGNORECASE | re.ASCII,
@@ -295,6 +307,7 @@ def urlize_ietf_docs(string, autoescape=None):
     )
     return mark_safe(string)
 
+
 urlize_ietf_docs = stringfilter(urlize_ietf_docs)
 
 @register.filter(name='urlize_related_source_list', is_safe=True, needs_autoescape=True)
@@ -492,10 +505,8 @@ def ad_area(user):
 @register.filter
 def format_history_text(text, trunc_words=25):
     """Run history text through some cleaning and add ellipsis if it's too long."""
-    full = mark_safe(text)
-    if "</a>" not in full:
-        full = urlize_ietf_docs(full)
-    full = bleach_linker.linkify(full)
+    full = mark_safe(bleach_cleaner.clean(text))
+    full = bleach_linker.linkify(urlize_ietf_docs(full))
 
     return format_snippet(full, trunc_words)
 
@@ -840,7 +851,6 @@ def is_valid_url(url):
     """
     Check if the given URL is syntactically valid
     """
-    validate_url = URLValidator()
     try:
         validate_url(url)
     except ValidationError:

ietf/doc/templatetags/tests_ietf_filters.py

@@ -1,76 +1,147 @@
 # Copyright The IETF Trust 2022, All Rights Reserved
 
-from ietf.doc.templatetags.ietf_filters import urlize_ietf_docs
+from django.conf import settings
+
+from ietf.doc.factories import (
+    WgDraftFactory,
+    IndividualDraftFactory,
+    CharterFactory,
+    NewRevisionDocEventFactory,
+)
+from ietf.doc.models import State, DocEvent, DocAlias
+from ietf.doc.templatetags.ietf_filters import urlize_ietf_docs, is_valid_url
+from ietf.person.models import Person
 from ietf.utils.test_utils import TestCase
 
-import debug # pyflakes: ignore
+import debug  # pyflakes: ignore
+
 # TODO: most other filters need test cases, too
 
 
 class IetfFiltersTests(TestCase):
+    def test_is_valid_url(self):
+        cases = [(settings.IDTRACKER_BASE_URL, True), ("not valid", False)]
+        for url, result in cases:
+            self.assertEqual(is_valid_url(url), result)
+
     def test_urlize_ietf_docs(self):
+        wg_id = WgDraftFactory()
+        wg_id.set_state(State.objects.get(type="draft", slug="rfc"))
+        wg_id.std_level_id = "bcp"
+        wg_id.save_with_history(
+            [
+                DocEvent.objects.create(
+                    doc=wg_id,
+                    rev=wg_id.rev,
+                    type="published_rfc",
+                    by=Person.objects.get(name="(System)"),
+                )
+            ]
+        )
+        DocAlias.objects.create(name="rfc123456").docs.add(wg_id)
+        DocAlias.objects.create(name="bcp123456").docs.add(wg_id)
+        DocAlias.objects.create(name="std123456").docs.add(wg_id)
+        DocAlias.objects.create(name="fyi123456").docs.add(wg_id)
+
+        id = IndividualDraftFactory(name="draft-me-rfc123456bis")
+        id_num = IndividualDraftFactory(name="draft-rosen-rfcefdp-update-2026")
+        id_num_two = IndividualDraftFactory(name="draft-spaghetti-idr-deprecate-8-9-10")
+        id_plus = IndividualDraftFactory(name="draft-odell-8+8")
+        id_plus_end = IndividualDraftFactory(name="draft-durand-gse+")
+        id_dot = IndividualDraftFactory(name="draft-ietf-pem-ansix9.17")
+        charter = CharterFactory()
+        e = NewRevisionDocEventFactory(doc=charter, rev="01")
+        charter.rev = e.rev
+        charter.save_with_history([e])
+        e = NewRevisionDocEventFactory(doc=charter, rev="01-00")
+        charter.rev = e.rev
+        charter.save_with_history([e])
+
         cases = [
             ("no change", "no change"),
-            ("bcp1", '<a href="/doc/bcp1/">bcp1</a>'),
-            ("Std 003", '<a href="/doc/std3/">Std 003</a>'),
+            ("bCp123456", '<a href="/doc/bcp123456/">bCp123456</a>'),
+            ("Std 00123456", '<a href="/doc/std123456/">Std 00123456</a>'),
             (
-                "FYI02 changes Std 003",
-                '<a href="/doc/fyi2/">FYI02</a> changes <a href="/doc/std3/">Std 003</a>',
+                "FyI  0123456 changes std 00123456",
+                '<a href="/doc/fyi123456/">FyI  0123456</a> changes <a href="/doc/std123456/">std 00123456</a>',
             ),
-            ("rfc2119", '<a href="/doc/rfc2119/">rfc2119</a>'),
-            ("Rfc 02119", '<a href="/doc/rfc2119/">Rfc 02119</a>'),
-            ("draft-abc-123", '<a href="/doc/draft-abc-123/">draft-abc-123</a>'),
+            ("rfc123456", '<a href="/doc/rfc123456/">rfc123456</a>'),
+            ("Rfc 0123456", '<a href="/doc/rfc123456/">Rfc 0123456</a>'),
+            (wg_id.name, f'<a href="/doc/{wg_id.name}/">{wg_id.name}</a>'),
             (
-                "draft-ietf-rfc9999-bis-01.txt",
-                '<a href="/doc/draft-ietf-rfc9999-bis/01/">draft-ietf-rfc9999-bis-01.txt</a>',
+                f"{id.name}-{id.rev}.txt",
+                f'<a href="/doc/{id.name}/{id.rev}/">{id.name}-{id.rev}.txt</a>',
             ),
             (
-                "foo RFC 9999 draft-ietf-rfc9999-bis-01 bar",
-                'foo <a href="/doc/rfc9999/">RFC 9999</a> <a href="/doc/draft-ietf-rfc9999-bis/01/">draft-ietf-rfc9999-bis-01</a> bar',
+                f"foo RFC 123456 {id.name}-{id.rev} bar",
+                f'foo <a href="/doc/rfc123456/">RFC 123456</a> <a href="/doc/{id.name}/{id.rev}/">{id.name}-{id.rev}</a> bar',
             ),
             (
-                "New version available: <b>draft-bryan-sipping-p2p-03.txt</b>",
-                'New version available: <b><a href="/doc/draft-bryan-sipping-p2p/03/">draft-bryan-sipping-p2p-03.txt</a></b>',
+                f"New version available: <b>{id.name}-{id.rev}.txt</b>",
+                f'New version available: <b><a href="/doc/{id.name}/{id.rev}/">{id.name}-{id.rev}.txt</a></b>',
             ),
             (
-                "New version available: <b>charter-ietf-6man-04.txt</b>",
-                'New version available: <b><a href="/doc/charter-ietf-6man/04/">charter-ietf-6man-04.txt</a></b>'
+                f"New version available: <b>{charter.name}-{charter.rev}.txt</b>",
+                f'New version available: <b><a href="/doc/{charter.name}/{charter.rev}/">{charter.name}-{charter.rev}.txt</a></b>',
             ),
             (
-                "New version available: <b>charter-ietf-6man-03-07.txt</b>",
-                'New version available: <b><a href="/doc/charter-ietf-6man/03-07/">charter-ietf-6man-03-07.txt</a></b>'
+                f"New version available: <b>{charter.name}-01-00.txt</b>",
+                f'New version available: <b><a href="/doc/{charter.name}/01-00/">{charter.name}-01-00.txt</a></b>',
             ),
             (
                 "repository https://github.com/tlswg/draft-ietf-tls-ticketrequest",
-                'repository https://github.com/tlswg/draft-ietf-tls-ticketrequest'
+                "repository https://github.com/tlswg/draft-ietf-tls-ticketrequest",
+            ),
+            (
+                '<a href="mailto:draft-ietf-some-names@ietf.org">draft-ietf-some-names@ietf.org</a>',
+                '<a href="mailto:draft-ietf-some-names@ietf.org">draft-ietf-some-names@ietf.org</a>',
             ),
             (
-                "draft-madanapalli-nd-over-802.16-problems",
-                '<a href="/doc/draft-madanapalli-nd-over-802.16-problems/">draft-madanapalli-nd-over-802.16-problems</a>'
+                "http://ieee802.org/1/files/public/docs2015/cn-thaler-Qcn-draft-PAR.pdf",
+                "http://ieee802.org/1/files/public/docs2015/cn-thaler-Qcn-draft-PAR.pdf",
             ),
             (
-                 "draft-madanapalli-nd-over-802.16-problems-02.txt",
-                '<a href="/doc/draft-madanapalli-nd-over-802.16-problems/02/">draft-madanapalli-nd-over-802.16-problems-02.txt</a>'
+                f"{id_num.name}.pdf",
+                f'<a href="/doc/{id_num.name}/">{id_num.name}.pdf</a>',
             ),
             (
-                '<a href="mailto:draft-ietf-some-names@ietf.org">draft-ietf-some-names@ietf.org</a>',
-                '<a href="mailto:draft-ietf-some-names@ietf.org">draft-ietf-some-names@ietf.org</a>',
+                f"{id_num.name}-{id_num.rev}.txt",
+                f'<a href="/doc/{id_num.name}/{id_num.rev}/">{id_num.name}-{id_num.rev}.txt</a>',
             ),
             (
-                "http://ieee802.org/1/files/public/docs2015/cn-thaler-Qcn-draft-PAR.pdf",
-                "http://ieee802.org/1/files/public/docs2015/cn-thaler-Qcn-draft-PAR.pdf"
-            )
+                f"{id_num_two.name}.pdf",
+                f'<a href="/doc/{id_num_two.name}/">{id_num_two.name}.pdf</a>',
+            ),
+            (
+                f"{id_num_two.name}-{id_num_two.rev}.txt",
+                f'<a href="/doc/{id_num_two.name}/{id_num_two.rev}/">{id_num_two.name}-{id_num_two.rev}.txt</a>',
+            ),
+            (
+                f"{id_plus.name}",
+                f'<a href="/doc/{id_plus.name}/">{id_plus.name}</a>',
+            ),
+            (
+                f"{id_plus.name}-{id_plus.rev}.txt",
+                f'<a href="/doc/{id_plus.name}/{id_plus.rev}/">{id_plus.name}-{id_plus.rev}.txt</a>',
+            ),
+            (
+                f"{id_plus_end.name}",
+                f'<a href="/doc/{id_plus_end.name}/">{id_plus_end.name}</a>',
+            ),
+            (
+                f"{id_plus_end.name}-{id_plus_end.rev}.txt",
+                f'<a href="/doc/{id_plus_end.name}/{id_plus_end.rev}/">{id_plus_end.name}-{id_plus_end.rev}.txt</a>',
+            ),
+            (
+                f"{id_dot.name}",
+                f'<a href="/doc/{id_dot.name}/">{id_dot.name}</a>',
+            ),
+            (
+                f"{id_dot.name}-{id_dot.rev}.txt",
+                f'<a href="/doc/{id_dot.name}/{id_dot.rev}/">{id_dot.name}-{id_dot.rev}.txt</a>',
+            ),
         ]
 
-        # Some edge cases scraped from existing old draft names
-        for name in [
-            # "draft-odell-8+8", # This fails since + matches the right side of \b
-            # "draft-durand-gse+", # same failure
-            "draft-kim-xcast+-few-2-few",
-            #"draft-ietf-pem-ansix9.17",  # Fails because of not being greedy with . before txt
-        ]:
-            cases.append((name,f'<a href="/doc/{name}/">{name}</a>'))
-
         for input, output in cases:
             #debug.show("(urlize_ietf_docs(input),output)")
             self.assertEqual(urlize_ietf_docs(input), output)