From 568e92aaa9140e416497e09f40cf277c8aa48006 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 4 May 2022 13:48:55 +0300 Subject: [PATCH 01/20] Unicode messages are triggered by both db content and tests --- ietf/utils/test_runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index eb380df6f33..c4344155c07 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -183,8 +183,7 @@ def vnu_filter_message(msg, filter_db_issues, filter_test_issues): if filter_test_issues and re.search( r"""Ceci\ n'est\ pas\ une\ URL| ^The\ '\w+'\ attribute\ on\ the\ '\w+'\ element\ is\ obsolete| - ^Section\ lacks\ heading| - is\ not\ in\ Unicode\ Normalization\ Form\ C""", + ^Section\ lacks\ heading""", msg["message"], flags=re.VERBOSE, ): @@ -193,7 +192,8 @@ def vnu_filter_message(msg, filter_db_issues, filter_test_issues): return re.search( r"""document\ is\ not\ mappable\ to\ XML\ 1| ^Attribute\ 'required'\ not\ allowed\ on\ element\ 'div'| - ^The\ 'type'\ attribute\ is\ unnecessary\ for\ JavaScript""", + ^The\ 'type'\ attribute\ is\ unnecessary\ for\ JavaScript| + is\ not\ in\ Unicode\ Normalization\ Form\ C""", msg["message"], flags=re.VERBOSE, ) From 05be333da9a550cd6e5e53387ca5153a780856b8 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 4 May 2022 13:49:19 +0300 Subject: [PATCH 02/20] Make ids unique --- ietf/templates/meeting/agenda.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ietf/templates/meeting/agenda.html b/ietf/templates/meeting/agenda.html index d4c9566b214..e93e84a0506 100644 --- a/ietf/templates/meeting/agenda.html +++ b/ietf/templates/meeting/agenda.html @@ -140,13 +140,13 @@

{% if item.session_keyword %} From dbf41f83ac41e015503a55c02c5ee949c4a38228 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 4 May 2022 16:16:14 +0300 Subject: [PATCH 03/20] Avoid "No value found" message on page --- ietf/templates/doc/document_ballot_content.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ietf/templates/doc/document_ballot_content.html b/ietf/templates/doc/document_ballot_content.html index fb612f371e8..54f73d8d4f6 100644 --- a/ietf/templates/doc/document_ballot_content.html +++ b/ietf/templates/doc/document_ballot_content.html @@ -105,7 +105,7 @@
{{ p.pos.name }} - ({{ p.discuss_time|date:"Y-m-d" }}{% if not p.for_current_revision %}{% if p.discuss_time %} {% endif %}for -{{ p.get_dochistory.rev }}{% endif %}) + ({{ p.discuss_time|date:"Y-m-d" }}{% if not p.for_current_revision and p.get_dochistory.rev %}{% if p.discuss_time %} {% endif %}for -{{ p.get_dochistory.rev }}{% endif %}) {% if p.send_email %} @@ -131,7 +131,7 @@
Comment - ({{ p.comment_time|date:"Y-m-d" }}{% if not p.for_current_revision %}{% if p.comment_time %} {% endif %}for -{{ p.get_dochistory.rev }}{% endif %}) + ({{ p.comment_time|date:"Y-m-d" }}{% if not p.for_current_revision and p.get_dochistory.rev %}{% if p.comment_time %} {% endif %}for -{{ p.get_dochistory.rev }}{% endif %}) {% if p.send_email %} @@ -178,9 +178,9 @@ {{ p.pos.name }} {% if p.pos.blocking and p.discuss %} [Treat as non-blocking comment] - ({{ p.discuss_time|date:"Y-m-d" }}{% if not p.for_current_revision %}{% if p.discuss_time %} {% endif %}for -{{ p.get_dochistory.rev }}{% endif %}) + ({{ p.discuss_time|date:"Y-m-d" }}{% if not p.for_current_revision and p.get_dochistory.rev %}{% if p.discuss_time %} {% endif %}for -{{ p.get_dochistory.rev }}{% endif %}) {% else %} - ({{ p.comment_time|date:"Y-m-d" }}{% if not p.for_current_revision %}{% if p.comment_time %} {% endif %}for -{{ p.get_dochistory.rev }}{% endif %}) + ({{ p.comment_time|date:"Y-m-d" }}{% if not p.for_current_revision and p.get_dochistory.rev %}{% if p.comment_time %} {% endif %}for -{{ p.get_dochistory.rev }}{% endif %}) {% endif %} {% if p.send_email %} Date: Wed, 4 May 2022 16:39:27 +0300 Subject: [PATCH 04/20] Strip HTML from history entries, it's often broken --- ietf/doc/templatetags/ietf_filters.py | 8 +++----- ietf/utils/text.py | 9 +++++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index f162a6bbc21..c2209d154f8 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -29,7 +29,7 @@ from ietf.utils.html import sanitize_fragment from ietf.utils import log from ietf.doc.utils import prettify_std_name -from ietf.utils.text import wordwrap, fill, wrap_text_if_unwrapped, bleach_linker +from ietf.utils.text import wordwrap, fill, wrap_text_if_unwrapped, bleach_linker, bleach_cleaner register = template.Library() @@ -492,10 +492,8 @@ def ad_area(user): @register.filter def format_history_text(text, trunc_words=25): """Run history text through some cleaning and add ellipsis if it's too long.""" - full = mark_safe(text) - if "" not in full: - full = urlize_ietf_docs(full) - full = bleach_linker.linkify(full) + full = mark_safe(bleach_cleaner.clean(text)) + full = bleach_linker.linkify(urlize_ietf_docs(full)) return format_snippet(full, trunc_words) diff --git a/ietf/utils/text.py b/ietf/utils/text.py index 1ef35cc3136..555267b4676 100644 --- a/ietf/utils/text.py +++ b/ietf/utils/text.py @@ -2,7 +2,8 @@ # -*- coding: utf-8 -*- -import bleach +import bleach # type: ignore +import copy import email import re import textwrap @@ -17,7 +18,7 @@ from .texescape import init as texescape_init, tex_escape_map tlds_sorted = sorted(tlds.tld_set, key=len, reverse=True) -protocols = bleach.sanitizer.ALLOWED_PROTOCOLS +protocols = copy.copy(bleach.sanitizer.ALLOWED_PROTOCOLS) protocols.append("ftp") # we still have some ftp links bleach_linker = bleach.Linker( url_re=bleach.linkifier.build_url_re(tlds=tlds_sorted, protocols=protocols), @@ -25,6 +26,10 @@ parse_email=True ) +tags = copy.copy(bleach.sanitizer.ALLOWED_TAGS) +tags.remove("a") +bleach_cleaner = bleach.sanitizer.Cleaner(tags=tags, protocols=protocols) + @keep_lazy(str) def xslugify(value): From c2e62265799b84457e43e4c4ce1dac0b22fde37a Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 4 May 2022 16:45:37 +0300 Subject: [PATCH 05/20] Check HTML sources for occurrences of "** No value found for" and fix them --- bin/test-crawl | 4 ++-- ietf/templates/group/group_about.html | 4 +++- ietf/templates/ipr/details_view.html | 4 ++-- ietf/utils/test_runner.py | 5 +++++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/bin/test-crawl b/bin/test-crawl index 8ad475aead7..022f6061881 100755 --- a/bin/test-crawl +++ b/bin/test-crawl @@ -175,7 +175,7 @@ def check_html_valid(url, response, args): assert ret for m in json.loads(ret)["messages"]: if "lastLine" not in m: - tag = m # just dump the raw JSON for now + tag = m["message"] else: tag = vnu_fmt_message(url, m, content.decode()) # disregard some HTML issues that are (usually) due to invalid @@ -211,7 +211,7 @@ def skip_url(url): r"^/wg/[a-z0-9-]+/deps/svg/", # Skip other bad urls r"^/dir/tsvdir/reviews/", - r"^/ipr/\d{,3}/history/", + # r"^/ipr/\d{,3}/history/", # Skip most html conversions, not worth the time r"^/doc/html/draft-[0-9ac-z]", r"^/doc/html/draft-b[0-9b-z]", diff --git a/ietf/templates/group/group_about.html b/ietf/templates/group/group_about.html index f16fb6b27c8..41e524c8c5a 100644 --- a/ietf/templates/group/group_about.html +++ b/ietf/templates/group/group_about.html @@ -68,7 +68,9 @@ {% if group.charter %} {{ group.charter.name }}-{{ group.charter.rev }} - {{ group.charter.get_state.name }} + {% if group.charter.get_state.name %} + {{ group.charter.get_state.name }} + {% endif %} {% else %} (None) {% if user|has_role:"Area Director,Secretariat" %} diff --git a/ietf/templates/ipr/details_view.html b/ietf/templates/ipr/details_view.html index c98329b9f23..3188f06b9a5 100644 --- a/ietf/templates/ipr/details_view.html +++ b/ietf/templates/ipr/details_view.html @@ -5,7 +5,7 @@ {% block title %}IPR Details - {{ ipr.title }}{% endblock %} {% block pagehead %} + content="IPR disclosure{% if ipr.ipr_id %} #{{ ipr.ipr_id }}{% endif %}: {{ ipr.title }} ({{ ipr.time|date:'Y' }})"> {% endblock %} {% block content %} {% origin %} @@ -791,4 +791,4 @@

Only those sections of the relevant entry form where the submitter provided information are displayed above.

-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index c4344155c07..200ce047ceb 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -126,6 +126,11 @@ def start_vnu_server(port=8888): def vnu_validate(html, content_type="text/html", port=8888): "Pass the HTML to the vnu server running on the indicated port" + if "** No value found for " in html.decode(): + return json.dumps( + {"messages": [{"message": '"** No value found for" in source'}]} + ) + gzippeddata = gzip.compress(html) try: req = http.request( From 1bcb04cb26d19d2ac191da759b76f25ca6b72236 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 4 May 2022 17:32:18 +0300 Subject: [PATCH 06/20] Fix another occurrence of "** No value found for" --- ietf/templates/meeting/agenda.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ietf/templates/meeting/agenda.html b/ietf/templates/meeting/agenda.html index e93e84a0506..ea18c9579ec 100644 --- a/ietf/templates/meeting/agenda.html +++ b/ietf/templates/meeting/agenda.html @@ -268,7 +268,9 @@

{% endif %} -
{{ item.session.historic_group.historic_parent.acronym }}
+ {% if item.session.historic_group.historic_parent.acronym %} +
{{ item.session.historic_group.historic_parent.acronym }}
+ {% endif %} {% if item.session.historic_group %} From c396eef654d0e6149860f0b7500dbd246beacb94 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 5 May 2022 09:12:32 +0300 Subject: [PATCH 07/20] Fix more occurrences of "** No value found for" --- ietf/templates/doc/document_material.html | 8 ++- ietf/templates/ipr/details_view.html | 78 ++++++++++++++++------- 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/ietf/templates/doc/document_material.html b/ietf/templates/doc/document_material.html index ca188282901..e1dc91e449b 100644 --- a/ietf/templates/doc/document_material.html +++ b/ietf/templates/doc/document_material.html @@ -56,7 +56,13 @@ {% doc_edit_button 'ietf.doc.views_material.edit_material' name=doc.name action="state" %} {% endif %} - {{ doc.get_state.name }} + + {% if doc.get_state.name %} + {{ doc.get_state.name }} + {% else %} + (None) + {% endif %} + {% if other_types %} diff --git a/ietf/templates/ipr/details_view.html b/ietf/templates/ipr/details_view.html index 3188f06b9a5..bdda164a629 100644 --- a/ietf/templates/ipr/details_view.html +++ b/ietf/templates/ipr/details_view.html @@ -170,7 +170,13 @@

{% endif %} - {% if prev %}{{ prev.holder_legal_name }}{% endif %} + {% if prev %} + + {% if prev.holder_legal_name %} + {{ prev.holder_legal_name }} + {% endif %} + + {% endif %} Holder legal name {{ ipr.holder_legal_name }} @@ -207,7 +213,9 @@

{% cycle section %}. Patent Holder's Contact for License Applic {% if prev %} - {{ prev.holder_contact_name }} + {% if prev.holder_contact_name %} + {{ prev.holder_contact_name }} + {% endif %} {% endif %} @@ -220,7 +228,9 @@

{% cycle section %}. Patent Holder's Contact for License Applic {% if prev %} - {{ prev.holder_contact_email|linkify }} + {% if prev.holder_contact_email %} + {{ prev.holder_contact_email|linkify }} + {% endif %} {% endif %} @@ -233,14 +243,16 @@

{% cycle section %}. Patent Holder's Contact for License Applic {% if prev %} - {{ prev.holder_contact_info|linebreaks }} + {% if prev.holder_contact_info %} + {{ prev.holder_contact_info|linkify|linebreaks }} + {% endif %} {% endif %} Holder contact info - {% if ipr.holder_contact_info %}{{ ipr.holder_contact_info|linebreaks }}{% endif %} + {% if ipr.holder_contact_info %}{{ ipr.holder_contact_info|linkify|linebreaks }}{% endif %} @@ -282,7 +294,9 @@

{% if prev %} - {{ prev.ietfer_name }} + {% if prev.ietfer_name %} + {{ prev.ietfer_name }} + {% endif %} {% endif %} @@ -295,7 +309,9 @@

{% if prev %} - {{ prev.ietfer_contact_email|linkify }} + {% if prev.ietfer_contact_email %} + {{ prev.ietfer_contact_email|linkify }} + {% endif %} {% endif %} @@ -308,14 +324,16 @@

{% if prev %} - {{ prev.ietfer_contact_info|linebreaks }} + {% if prev.ietfer_contact_info %} + {{ prev.ietfer_contact_info|linkify|linebreaks }} + {% endif %} {% endif %} Other info - {{ ipr.ietfer_contact_info|linebreaks }} + {{ ipr.ietfer_contact_info|linkify|linebreaks }} @@ -474,7 +492,9 @@

{% if prev %} - {{ prev.patent_info|linebreaks }} + {% if prev.patent_info %} + {{ prev.patent_info|urlize_ietf_docs|linkify|linebreaks }} + {% endif %} {% endif %} @@ -483,7 +503,7 @@

or Application/File number(s) - {{ ipr.patent_info|linebreaks }} + {{ ipr.patent_info|urlize_ietf_docs|linkify|linebreaks }} @@ -523,7 +543,9 @@

{% if prev %} - {{ prev.has_patent_pending|yesno:"Yes,No" }} + {% if prev.has_patent_pending %} + {{ prev.has_patent_pending|yesno:"Yes,No" }} + {% endif %} {% endif %} @@ -592,10 +614,12 @@

{% if prev %} - {% if prev.licensing.slug == "provided-later" %} - {{ prev.licensing.desc|slice:"2:"|slice:":117" }}) - {% else %} - {{ prev.licensing.desc|slice:"2:" }} + {% if prev.licensing.slug %} + {% if prev.licensing.slug == "provided-later" %} + {{ prev.licensing.desc|slice:"2:"|slice:":117" }}) + {% else %} + {{ prev.licensing.desc|slice:"2:" }} + {% endif %} {% endif %} {% endif %} @@ -613,7 +637,9 @@

{% if prev %} - {{ prev.licensing_comments|default:"(No information submitted)"|linebreaks }} + {% if prev.licensing_comments %} + {{ prev.licensing_comments|default:"(No information submitted)"|urlize_ietf_docs|linkify|linebreaks }} + {% endif %} {% endif %} @@ -666,14 +692,16 @@

{% if prev %} - {{ prev.statement|linebreaks }} + {% if prev.statement %} + {{ prev.statement|urlize_ietf_docs|linkify|linebreaks }} + {% endif %} {% endif %} Statement - {{ ipr.statement|linebreaks }} + {{ ipr.statement|urlize_ietf_docs|linkify|linebreaks }} @@ -714,7 +742,9 @@

{% if prev %} - {{ prev.submitter_name }} + {% if prev.submitter_name %} + {{ prev.submitter_name }} + {% endif %} {% endif %} @@ -727,7 +757,9 @@

{% if prev %} - {{ prev.submitter_email|linkify }} + {% if prev.submitter_email %} + {{ prev.submitter_email|linkify }} + {% endif %} {% endif %} @@ -775,7 +807,9 @@

{% if prev %} - {{ prev.notes|urlize_ietf_docs|linkify|linebreaks }} + {% if prev.notes %} + {{ prev.notes|urlize_ietf_docs|linkify|linebreaks }} + {% endif %} {% endif %} From 0e974b13a551eb4160a3a3be0e6c041478bafe42 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 5 May 2022 16:47:25 +0300 Subject: [PATCH 08/20] Fix document revision stripping --- ietf/doc/templatetags/ietf_filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index c2209d154f8..5151f68846a 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -216,7 +216,7 @@ def find_unique(n): # check for embedded rev - this may be ambiguous, so don't # chop it off if we don't find a match - rev_split = re.search("^(.+)-([0-9]{2,})$", name) + rev_split = re.search(r"^(.+)-(\d{2}|[1-9]\d{2,})$", name) if rev_split: name = rev_split.group(1) if find_unique(name): From b4f6556c9d4abf12d952bd82e4a4b4ee3beb8b1e Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 5 May 2022 16:47:39 +0300 Subject: [PATCH 09/20] Force breaks of long (garbage) words --- ietf/templates/ipr/details_history.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/templates/ipr/details_history.html b/ietf/templates/ipr/details_history.html index aa1ea8f8313..b8fecf78074 100644 --- a/ietf/templates/ipr/details_history.html +++ b/ietf/templates/ipr/details_history.html @@ -49,7 +49,7 @@

{% endif %} {% person_link e.by %} - + {% if e.message %} {% if e.response_due %}Response due {{ e.response_due|date:"Y-m-d" }}{% endif %} {# FIXME: can't do format_history_text, because that inserts a
into the 
, which is illegal. Need to rework the snippeting. #}

From f07a472ce53309eb01799d161cafec3e0529b688 Mon Sep 17 00:00:00 2001
From: Lars Eggert 
Date: Thu, 5 May 2022 20:12:49 +0300
Subject: [PATCH 10/20] Check URL validity before urlizing them

---
 ietf/utils/text.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ietf/utils/text.py b/ietf/utils/text.py
index 555267b4676..9596d192d66 100644
--- a/ietf/utils/text.py
+++ b/ietf/utils/text.py
@@ -10,6 +10,8 @@
 import tlds
 import unicodedata
 
+from django.core.validators import URLValidator
+from django.core.exceptions import ValidationError
 from django.utils.functional import keep_lazy
 from django.utils.safestring import mark_safe
 
@@ -20,7 +22,21 @@
 tlds_sorted = sorted(tlds.tld_set, key=len, reverse=True)
 protocols = copy.copy(bleach.sanitizer.ALLOWED_PROTOCOLS)
 protocols.append("ftp")  # we still have some ftp links
+validate_url = URLValidator()
+
+
+def check_url_validity(attrs, new=False):
+    url = attrs[(None, 'href')]
+    try:
+        if url.startswith("http"):
+            validate_url(url)
+    except ValidationError:
+        return None
+    return attrs
+
+
 bleach_linker = bleach.Linker(
+    callbacks=[check_url_validity],
     url_re=bleach.linkifier.build_url_re(tlds=tlds_sorted, protocols=protocols),
     email_re=bleach.linkifier.build_email_re(tlds=tlds_sorted), # type: ignore
     parse_email=True

From 5316742bcf7c4eaa930734ec8415762bcce72ed0 Mon Sep 17 00:00:00 2001
From: Lars Eggert 
Date: Thu, 5 May 2022 20:13:22 +0300
Subject: [PATCH 11/20] Handle some additional corner cases

---
 ietf/doc/templatetags/ietf_filters.py | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py
index 5151f68846a..23f2685d045 100644
--- a/ietf/doc/templatetags/ietf_filters.py
+++ b/ietf/doc/templatetags/ietf_filters.py
@@ -19,7 +19,6 @@
 from django.utils.encoding import force_str # pyflakes:ignore force_str is used in the doctests
 from django.urls import reverse as urlreverse
 from django.core.cache import cache
-from django.core.validators import URLValidator
 from django.core.exceptions import ValidationError
 
 import debug                            # pyflakes:ignore
@@ -29,7 +28,7 @@
 from ietf.utils.html import sanitize_fragment
 from ietf.utils import log
 from ietf.doc.utils import prettify_std_name
-from ietf.utils.text import wordwrap, fill, wrap_text_if_unwrapped, bleach_linker, bleach_cleaner
+from ietf.utils.text import wordwrap, fill, wrap_text_if_unwrapped, bleach_linker, bleach_cleaner, validate_url
 
 register = template.Library()
 
@@ -189,8 +188,8 @@ def rfceditor_info_url(rfcnum : str):
     return urljoin(settings.RFC_EDITOR_INFO_BASE_URL, f'rfc{rfcnum}')
 
 
-def doc_exists(name):
-    """Check whether a given document exists"""
+def doc_canonical_name(name):
+    """Check whether a given document exists, and return its canonical name"""
     def find_unique(n):
         key = hash(n)
         found = cache.get(key)
@@ -204,7 +203,7 @@ def find_unique(n):
     if settings.SERVER_MODE == 'test':
         # unless we are running test-crawl, which would otherwise 404
         if "DJANGO_URLIZE_IETF_DOCS_PRODUCTION" not in os.environ:
-            return True
+            return name
 
     # chop away extension
     extension_split = re.search(r"^(.+)\.(txt|ps|pdf)$", name)
@@ -212,7 +211,7 @@ def find_unique(n):
         name = extension_split.group(1)
 
     if find_unique(name):
-        return True
+        return name
 
     # check for embedded rev - this may be ambiguous, so don't
     # chop it off if we don't find a match
@@ -220,27 +219,28 @@ def find_unique(n):
     if rev_split:
         name = rev_split.group(1)
         if find_unique(name):
-            return True
+            return name
 
-    return False
+    return ""
 
 
 def link_charter_doc_match1(match):
-    if not doc_exists(match[0]):
+    if not doc_canonical_name(match[0]):
         return match[0]
     return f'{match[0]}'
 
 
 def link_charter_doc_match2(match):
-    if not doc_exists(match[0]):
+    if not doc_canonical_name(match[0]):
         return match[0]
     return f'{match[0]}'
 
 
 def link_non_charter_doc_match(match):
-    if not doc_exists(match[0]):
+    name = doc_canonical_name(match[0])
+    if not name:
         return match[0]
-    if len(match[3]) == 2 and match[3].isdigit():
+    if name != match[0] and len(match[3]) == 2 and match[3].isdigit():
         return f'{match[0]}'
     else:
         return f'{match[0]}'
@@ -249,7 +249,7 @@ def link_non_charter_doc_match(match):
 def link_other_doc_match(match):
     # there may be whitespace in the match
     doc = re.sub(r"\s+", "", match[0])
-    if not doc_exists(doc):
+    if not doc_canonical_name(doc):
         return match[0]
     return f'{match[1]}'
 
@@ -838,7 +838,6 @@ def is_valid_url(url):
     """
     Check if the given URL is syntactically valid
     """
-    validate_url = URLValidator()
     try:
         validate_url(url)
     except ValidationError:

From ae23718a702c5d8ee6c6a5c1acabe662d88c210b Mon Sep 17 00:00:00 2001
From: Lars Eggert 
Date: Thu, 5 May 2022 20:13:51 +0300
Subject: [PATCH 12/20] Linkify action items

---
 ietf/templates/iesg/agenda.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/templates/iesg/agenda.html b/ietf/templates/iesg/agenda.html
index 9cfab0b1ae1..629c0946f4f 100644
--- a/ietf/templates/iesg/agenda.html
+++ b/ietf/templates/iesg/agenda.html
@@ -20,7 +20,7 @@ 
{{ num }} {{ section.title|safe }}{{ num }} {{ section.title|safe }}

         {% endif %}
-        {% if num == "1.4" %}
{{ section.text }}
{% endif %}
+        {% if num == "1.4" %}
{{ section.text|urlize_ietf_docs|linkify }}
{% endif %}
         {% if num >= "2" and num < "5" %}
             {% if num == "2" %}
                 


From f7058d63a9b67effff63bdf970e40bb65fe5691e Mon Sep 17 00:00:00 2001
From: Lars Eggert 
Date: Thu, 5 May 2022 20:27:54 +0300
Subject: [PATCH 13/20] Don't create profile/email links for System

---
 ietf/templates/person/person_link.html | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ietf/templates/person/person_link.html b/ietf/templates/person/person_link.html
index 7264bead983..a8e2da9d773 100644
--- a/ietf/templates/person/person_link.html
+++ b/ietf/templates/person/person_link.html
@@ -1,3 +1,6 @@
+{% if email and email == "system@datatracker.ietf.org" or name and name == "(System)" %}
+    (System)
+{% else %}
 {% if email or name %}
-                {% endif %}{% else %}(None){% endif %}
\ No newline at end of file
+                {% endif %}{% else %}(None){% endif %}
+{% endif %}

From dfb1ffcfa0ddf74616a3386df9cd3381b50802bd Mon Sep 17 00:00:00 2001
From: Lars Eggert 
Date: Thu, 5 May 2022 21:00:18 +0300
Subject: [PATCH 14/20] Handle headings with HTML elements in them better

---
 ietf/static/js/ietf.js | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/ietf/static/js/ietf.js b/ietf/static/js/ietf.js
index 503a93b97ef..ef1acb79be3 100644
--- a/ietf/static/js/ietf.js
+++ b/ietf/static/js/ietf.js
@@ -199,12 +199,15 @@ $(function () {
                 .find("h1:visible, h2:visible, h3:visible, h4:visible, h5:visible, h6:visible, .nav-heading:visible")
                 .not(".navskip")
                 .each(function () {
-                    // Some headings have complex HTML in them - only use first part in that case.
-                    const text = $(this)
+                    // Some headings a line break have complex HTML in them - only use first part in that case.
+                    const frag = $(this)
                         .html()
-                        .split("<")
-                        .shift()
-                        .trim();
+                        .split(" $(x)
+                            .text())
+                        .join(" ");
 
                     if (text === undefined || text === "") {
                         // Nothing to do for empty headings.

From 9b19aa766ec3930bd4468c747b569f90e9fd42f1 Mon Sep 17 00:00:00 2001
From: Lars Eggert 
Date: Thu, 5 May 2022 21:01:23 +0300
Subject: [PATCH 15/20] Fix comment

---
 ietf/static/js/ietf.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/static/js/ietf.js b/ietf/static/js/ietf.js
index ef1acb79be3..f8cf82d9bf6 100644
--- a/ietf/static/js/ietf.js
+++ b/ietf/static/js/ietf.js
@@ -199,7 +199,7 @@ $(function () {
                 .find("h1:visible, h2:visible, h3:visible, h4:visible, h5:visible, h6:visible, .nav-heading:visible")
                 .not(".navskip")
                 .each(function () {
-                    // Some headings a line break have complex HTML in them - only use first part in that case.
+                    // Some headings have line breaks in them - only use first line in that case.
                     const frag = $(this)
                         .html()
                         .split("
Date: Fri, 6 May 2022 16:43:08 +0300
Subject: [PATCH 16/20] Fix another occurrence of "** No value found for"

---
 ietf/templates/doc/document_review.html | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/ietf/templates/doc/document_review.html b/ietf/templates/doc/document_review.html
index bf249ff53b8..113c407fe59 100644
--- a/ietf/templates/doc/document_review.html
+++ b/ietf/templates/doc/document_review.html
@@ -25,7 +25,13 @@
                         
                         State
                         
-                        {{ doc.get_state.name }}
+                        
+                            {% if doc.get_state.name %}
+                                {{ doc.get_state.name }}
+                            {% else %}
+                                (None)
+                            {% endif %}
+                        
                     
                 {% endif %}
                 {% if other_reviews %}

From a8b25cd085d0d007365d40307cfce1a9c32026da Mon Sep 17 00:00:00 2001
From: Lars Eggert 
Date: Fri, 6 May 2022 16:50:56 +0300
Subject: [PATCH 17/20] Better I-D URLization that handles more edge cases.
 Also, test for them.

---
 bin/test-crawl                              |   1 -
 ietf/doc/templatetags/ietf_filters.py       |  52 +++++---
 ietf/doc/templatetags/tests_ietf_filters.py | 127 ++++++++++++++------
 3 files changed, 122 insertions(+), 58 deletions(-)

diff --git a/bin/test-crawl b/bin/test-crawl
index 022f6061881..20dc1f0696b 100755
--- a/bin/test-crawl
+++ b/bin/test-crawl
@@ -44,7 +44,6 @@ args = parser.parse_args()
 
 # Import Django, call setup()
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.settings or "ietf.settings_testcrawl")
-os.environ["DJANGO_URLIZE_IETF_DOCS_PRODUCTION"] = "1"
 
 import django
 import django.test
diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py
index 23f2685d045..2770e7fb9f4 100644
--- a/ietf/doc/templatetags/ietf_filters.py
+++ b/ietf/doc/templatetags/ietf_filters.py
@@ -4,7 +4,6 @@
 
 import datetime
 import re
-import os
 from urllib.parse import urljoin
 
 from email.utils import parseaddr
@@ -190,6 +189,7 @@ def rfceditor_info_url(rfcnum : str):
 
 def doc_canonical_name(name):
     """Check whether a given document exists, and return its canonical name"""
+
     def find_unique(n):
         key = hash(n)
         found = cache.get(key)
@@ -199,14 +199,8 @@ def find_unique(n):
             cache.set(key, found)
         return None if found == "_" else found
 
-    # all documents exist when tests are running
-    if settings.SERVER_MODE == 'test':
-        # unless we are running test-crawl, which would otherwise 404
-        if "DJANGO_URLIZE_IETF_DOCS_PRODUCTION" not in os.environ:
-            return name
-
     # chop away extension
-    extension_split = re.search(r"^(.+)\.(txt|ps|pdf)$", name)
+    extension_split = re.search(r"^(.+)\.(txt|ps|pdf|html)$", name)
     if extension_split:
         name = extension_split.group(1)
 
@@ -237,21 +231,37 @@ def link_charter_doc_match2(match):
 
 
 def link_non_charter_doc_match(match):
-    name = doc_canonical_name(match[0])
-    if not name:
+    name = match[0]
+    cname = doc_canonical_name(name)
+    if not cname:
         return match[0]
-    if name != match[0] and len(match[3]) == 2 and match[3].isdigit():
-        return f'{match[0]}'
+    if name == cname:
+        return f'{match[0]}'
+
+    # if we get here, the name probably has a version number and/or extension at the end
+    rev_split = re.search(r"^(" + re.escape(cname) + r")-(\d{2,})", name)
+    if rev_split:
+        name = rev_split.group(1)
     else:
-        return f'{match[0]}'
+        return f'{match[0]}'
+
+    cname = doc_canonical_name(name)
+    if not cname:
+        return match[0]
+    if name == cname:
+        return f'{match[0]}'
+
+    # if we get here, we can't linkify
+    print("d")
+    return match[0]
 
 
 def link_other_doc_match(match):
-    # there may be whitespace in the match
-    doc = re.sub(r"\s+", "", match[0])
-    if not doc_canonical_name(doc):
+    doc = match[2].strip().lower()
+    rev = match[3]
+    if not doc_canonical_name(doc + rev):
         return match[0]
-    return f'{match[1]}'
+    return f'{match[1]}'
 
 
 @register.filter(name="urlize_ietf_docs", is_safe=True, needs_autoescape=True)
@@ -264,8 +274,8 @@ def urlize_ietf_docs(string, autoescape=None):
             string = escape(string)
         else:
             string = mark_safe(string)
-    exp1 = r"\b(?bcp1'),
-            ("Std 003", 'Std 003'),
+            ("bCp123456", 'bCp123456'),
+            ("Std 00123456", 'Std 00123456'),
             (
-                "FYI02 changes Std 003",
-                'FYI02 changes Std 003',
+                "FyI  0123456 changes std 00123456",
+                'FyI  0123456 changes std 00123456',
             ),
-            ("rfc2119", 'rfc2119'),
-            ("Rfc 02119", 'Rfc 02119'),
-            ("draft-abc-123", 'draft-abc-123'),
+            ("rfc123456", 'rfc123456'),
+            ("Rfc 0123456", 'Rfc 0123456'),
+            (wg_id.name, f'{wg_id.name}'),
             (
-                "draft-ietf-rfc9999-bis-01.txt",
-                'draft-ietf-rfc9999-bis-01.txt',
+                f"{id.name}-{id.rev}.txt",
+                f'{id.name}-{id.rev}.txt',
             ),
             (
-                "foo RFC 9999 draft-ietf-rfc9999-bis-01 bar",
-                'foo RFC 9999 draft-ietf-rfc9999-bis-01 bar',
+                f"foo RFC 123456 {id.name}-{id.rev} bar",
+                f'foo RFC 123456 {id.name}-{id.rev} bar',
             ),
             (
-                "New version available: draft-bryan-sipping-p2p-03.txt",
-                'New version available: draft-bryan-sipping-p2p-03.txt',
+                f"New version available: {id.name}-{id.rev}.txt",
+                f'New version available: {id.name}-{id.rev}.txt',
             ),
             (
-                "New version available: charter-ietf-6man-04.txt",
-                'New version available: charter-ietf-6man-04.txt'
+                f"New version available: {charter.name}-{charter.rev}.txt",
+                f'New version available: {charter.name}-{charter.rev}.txt',
             ),
             (
-                "New version available: charter-ietf-6man-03-07.txt",
-                'New version available: charter-ietf-6man-03-07.txt'
+                f"New version available: {charter.name}-{charter.rev}.txt",
+                f'New version available: {charter.name}-{charter.rev}.txt',
             ),
             (
                 "repository https://github.com/tlswg/draft-ietf-tls-ticketrequest",
-                'repository https://github.com/tlswg/draft-ietf-tls-ticketrequest'
+                "repository https://github.com/tlswg/draft-ietf-tls-ticketrequest",
             ),
             (
-                "draft-madanapalli-nd-over-802.16-problems",
-                'draft-madanapalli-nd-over-802.16-problems'
+                'draft-ietf-some-names@ietf.org',
+                'draft-ietf-some-names@ietf.org',
             ),
             (
-                 "draft-madanapalli-nd-over-802.16-problems-02.txt",
-                'draft-madanapalli-nd-over-802.16-problems-02.txt'
+                "http://ieee802.org/1/files/public/docs2015/cn-thaler-Qcn-draft-PAR.pdf",
+                "http://ieee802.org/1/files/public/docs2015/cn-thaler-Qcn-draft-PAR.pdf",
             ),
             (
-                'draft-ietf-some-names@ietf.org',
-                'draft-ietf-some-names@ietf.org',
+                f"{id_num.name}.pdf",
+                f'{id_num.name}.pdf',
             ),
             (
-                "http://ieee802.org/1/files/public/docs2015/cn-thaler-Qcn-draft-PAR.pdf",
-                "http://ieee802.org/1/files/public/docs2015/cn-thaler-Qcn-draft-PAR.pdf"
-            )
+                f"{id_num.name}-{id_num.rev}.txt",
+                f'{id_num.name}-{id_num.rev}.txt',
+            ),
+            (
+                f"{id_num_two.name}.pdf",
+                f'{id_num_two.name}.pdf',
+            ),
+            (
+                f"{id_num_two.name}-{id_num_two.rev}.txt",
+                f'{id_num_two.name}-{id_num_two.rev}.txt',
+            ),
+            (
+                f"{id_plus.name}",
+                f'{id_plus.name}',
+            ),
+            (
+                f"{id_plus.name}-{id_plus.rev}.txt",
+                f'{id_plus.name}-{id_plus.rev}.txt',
+            ),
+            (
+                f"{id_plus_end.name}",
+                f'{id_plus_end.name}',
+            ),
+            (
+                f"{id_plus_end.name}-{id_plus_end.rev}.txt",
+                f'{id_plus_end.name}-{id_plus_end.rev}.txt',
+            ),
+            (
+                f"{id_dot.name}",
+                f'{id_dot.name}',
+            ),
+            (
+                f"{id_dot.name}-{id_dot.rev}.txt",
+                f'{id_dot.name}-{id_dot.rev}.txt',
+            ),
         ]
 
-        # Some edge cases scraped from existing old draft names
-        for name in [
-            # "draft-odell-8+8", # This fails since + matches the right side of \b
-            # "draft-durand-gse+", # same failure
-            "draft-kim-xcast+-few-2-few",
-            #"draft-ietf-pem-ansix9.17",  # Fails because of not being greedy with . before txt
-        ]:
-            cases.append((name,f'{name}'))
-
         for input, output in cases:
             #debug.show("(urlize_ietf_docs(input),output)")
             self.assertEqual(urlize_ietf_docs(input), output)

From 9e791d8d0bd31c70bf5f419f8b9dfb0f719ba596 Mon Sep 17 00:00:00 2001
From: Lars Eggert 
Date: Fri, 6 May 2022 17:12:35 +0300
Subject: [PATCH 18/20] Remove print

---
 ietf/doc/templatetags/ietf_filters.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py
index 2770e7fb9f4..f265ce82ad3 100644
--- a/ietf/doc/templatetags/ietf_filters.py
+++ b/ietf/doc/templatetags/ietf_filters.py
@@ -252,7 +252,6 @@ def link_non_charter_doc_match(match):
         return f'{match[0]}'
 
     # if we get here, we can't linkify
-    print("d")
     return match[0]
 
 

From f62f17706d7b2a75b856d624c8914d2b783fd581 Mon Sep 17 00:00:00 2001
From: Lars Eggert 
Date: Mon, 9 May 2022 10:03:03 +0300
Subject: [PATCH 19/20] Handle charters better

---
 ietf/doc/templatetags/ietf_filters.py       |  4 +++-
 ietf/doc/templatetags/tests_ietf_filters.py | 26 +++++++++++++++++----
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py
index f265ce82ad3..47088ad04ab 100644
--- a/ietf/doc/templatetags/ietf_filters.py
+++ b/ietf/doc/templatetags/ietf_filters.py
@@ -209,7 +209,9 @@ def find_unique(n):
 
     # check for embedded rev - this may be ambiguous, so don't
     # chop it off if we don't find a match
-    rev_split = re.search(r"^(.+)-(\d{2}|[1-9]\d{2,})$", name)
+    rev_split = re.search(r"^(charter-.+)-(\d{2}-\d{2})$", name) or re.search(
+        r"^(.+)-(\d{2}|[1-9]\d{2,})$", name
+    )
     if rev_split:
         name = rev_split.group(1)
         if find_unique(name):
diff --git a/ietf/doc/templatetags/tests_ietf_filters.py b/ietf/doc/templatetags/tests_ietf_filters.py
index a4267a8075d..9c968701531 100644
--- a/ietf/doc/templatetags/tests_ietf_filters.py
+++ b/ietf/doc/templatetags/tests_ietf_filters.py
@@ -1,8 +1,15 @@
 # Copyright The IETF Trust 2022, All Rights Reserved
 
-from ietf.doc.factories import WgDraftFactory, IndividualDraftFactory, CharterFactory
+from django.conf import settings
+
+from ietf.doc.factories import (
+    WgDraftFactory,
+    IndividualDraftFactory,
+    CharterFactory,
+    NewRevisionDocEventFactory,
+)
 from ietf.doc.models import State, DocEvent, DocAlias
-from ietf.doc.templatetags.ietf_filters import urlize_ietf_docs
+from ietf.doc.templatetags.ietf_filters import urlize_ietf_docs, is_valid_url
 from ietf.person.models import Person
 from ietf.utils.test_utils import TestCase
 
@@ -12,6 +19,11 @@
 
 
 class IetfFiltersTests(TestCase):
+    def test_is_valid_url(self):
+        cases = [(settings.IDTRACKER_BASE_URL, True), ("not valid", False)]
+        for url, result in cases:
+            self.assertEqual(is_valid_url(url), result)
+
     def test_urlize_ietf_docs(self):
         wg_id = WgDraftFactory()
         wg_id.set_state(State.objects.get(type="draft", slug="rfc"))
@@ -38,6 +50,12 @@ def test_urlize_ietf_docs(self):
         id_plus_end = IndividualDraftFactory(name="draft-durand-gse+")
         id_dot = IndividualDraftFactory(name="draft-ietf-pem-ansix9.17")
         charter = CharterFactory()
+        e = NewRevisionDocEventFactory(doc=charter, rev="01")
+        charter.rev = e.rev
+        charter.save_with_history([e])
+        e = NewRevisionDocEventFactory(doc=charter, rev="01-00")
+        charter.rev = e.rev
+        charter.save_with_history([e])
 
         cases = [
             ("no change", "no change"),
@@ -67,8 +85,8 @@ def test_urlize_ietf_docs(self):
                 f'New version available: {charter.name}-{charter.rev}.txt',
             ),
             (
-                f"New version available: {charter.name}-{charter.rev}.txt",
-                f'New version available: {charter.name}-{charter.rev}.txt',
+                f"New version available: {charter.name}-01-00.txt",
+                f'New version available: {charter.name}-01-00.txt',
             ),
             (
                 "repository https://github.com/tlswg/draft-ietf-tls-ticketrequest",

From dbc3119f6144d053d6a74641aa28c94dd6cf1a78 Mon Sep 17 00:00:00 2001
From: Lars Eggert 
Date: Tue, 10 May 2022 11:56:27 +0300
Subject: [PATCH 20/20] Cache for one day

---
 ietf/doc/templatetags/ietf_filters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py
index 47088ad04ab..896d327e710 100644
--- a/ietf/doc/templatetags/ietf_filters.py
+++ b/ietf/doc/templatetags/ietf_filters.py
@@ -196,7 +196,7 @@ def find_unique(n):
         if not found:
             exact = DocAlias.objects.filter(name=n).first()
             found = exact.name if exact else "_"
-            cache.set(key, found)
+            cache.set(key, found, timeout=60*60*24)  # cache for one day
         return None if found == "_" else found
 
     # chop away extension