Added a new document tab for some documents, showing htmlized drafts and RFCs

using the htmlization code previously developed for tools.ietf.org.  As the
generation of the htmlized page is a bit too costly to do on the fly for
often-referenced drafts and RFCs, the part of each page which contains the
htmlized document is cached on file with a cache time of 2 weeks.

Changed all links which pointed to the htmlized version on tools to instead
point at the datatracker htmlized document.

Tweaked some URLs which didn't permit retrieval of intermediate-rev-charters.

Narrowed the pattern for document names to disallow dots in names, and
instead explicitly enumerated the few historical draftw with dots in the
name.

Added a file-system cache for the htmlized documents, and specified a
max_entries value for caches, overriding the default 300 entries.

Tweaked the code for new author email entries to provide a time if missing
in an updated entry.

Changed links in various email templates which pointed at tools.ietf.org
pages to instead point at datatracker pages, where appropriate.

Changed the search result rows to provide links to both the current meta-
information document pages (with a (i) info symbol) and to the new htmlized
document pages.
 - Legacy-Id: 13040

Author: levkowetz

ietf/doc/templatetags/ietf_filters.py

@@ -197,7 +197,7 @@ def rfclink(string):
     URL for that RFC.
     """
     string = str(string);
-    return "https://tools.ietf.org/html/rfc" + string;
+    return "/doc/html/rfc" + string;
 
 @register.filter(name='urlize_ietf_docs', is_safe=True, needs_autoescape=True)
 def urlize_ietf_docs(string, autoescape=None):

ietf/doc/tests.py

@@ -493,6 +493,15 @@ def test_document_draft(self):
         self.assertTrue("Show full document text" in unicontent(r))
         self.assertFalse("Deimos street" in unicontent(r))
 
+        r = self.client.get(urlreverse("ietf.doc.views_doc.document_html", kwargs=dict(name=draft.name)))
+        self.assertEqual(r.status_code, 200)
+        self.assertTrue("Versions:" in unicontent(r))
+        self.assertTrue("Deimos street" in unicontent(r))
+        q = PyQuery(r.content)
+        self.assertEqual(len(q('.rfcmarkup pre')), 4)
+        self.assertEqual(len(q('span.h1')), 2)
+        self.assertEqual(len(q('a[href]')), 116)
+
         # expired draft
         draft.set_state(State.objects.get(type="draft", slug="expired"))
 

ietf/doc/urls.py

@@ -59,13 +59,14 @@
     url(r'^stats/newrevisiondocevent/data/?$', views_stats.chart_data_newrevisiondocevent),
     url(r'^stats/person/(?P<id>[0-9]+)/drafts/conf/?$', views_stats.chart_conf_person_drafts),
     url(r'^stats/person/(?P<id>[0-9]+)/drafts/data/?$', views_stats.chart_data_person_drafts),
+    url(r'^html/%(name)s(?:-%(rev)s)?(\.txt|\.html)?$' % settings.URL_REGEXPS, views_doc.document_html),
 
     url(r'^all/$', views_search.index_all_drafts),
     url(r'^active/$', views_search.index_active_drafts),
     url(r'^select2search/(?P<model_name>(document|docalias))/(?P<doc_type>draft)/$', views_search.ajax_select2_search_docs),
 
-    url(r'^%(name)s/(?:%(rev)s/)?$' % settings.URL_REGEXPS, views_doc.document_main),
-    url(r'^%(name)s/(?:%(rev)s/)?bibtex/$' % settings.URL_REGEXPS, views_doc.document_bibtex),
+    url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main),
+    url(r'^%(name)s(?:/%(rev)s)?/bibtex/$' % settings.URL_REGEXPS, views_doc.document_bibtex),
     url(r'^%(name)s/history/$' % settings.URL_REGEXPS, views_doc.document_history),
     url(r'^%(name)s/writeup/$' % settings.URL_REGEXPS, views_doc.document_writeup),
     url(r'^%(name)s/email/$' % settings.URL_REGEXPS, views_doc.document_email),

ietf/doc/urls_charter.py

@@ -1,7 +1,5 @@
 # Copyright The IETF Trust 2011, All Rights Reserved
 
-from django.conf import settings
-
 from ietf.doc import views_charter, views_doc
 from ietf.utils.urls import url
 
@@ -17,5 +15,5 @@
     url(r'^ballotwriteupnotes/$', views_charter.ballot_writeupnotes),
     url(r'^approve/$', views_charter.approve),
     url(r'^submit/(?:(?P<option>initcharter|recharter)/)?$', views_charter.submit),
-    url(r'^withmilestones-%(rev)s.txt$' % settings.URL_REGEXPS, views_charter.charter_with_milestones_txt),
+    url(r'^withmilestones-(?P<rev>[0-9-]{2,5}).txt$', views_charter.charter_with_milestones_txt),
 ]

ietf/doc/utils.py

@@ -12,6 +12,8 @@
 from django.utils.html import escape
 from django.core.urlresolvers import reverse as urlreverse
 
+import debug                            # pyflakes:ignore
+
 from ietf.doc.models import Document, DocHistory, State, DocumentAuthor, DocHistoryAuthor
 from ietf.doc.models import DocAlias, RelatedDocument, RelatedDocHistory, BallotType, DocReminder
 from ietf.doc.models import DocEvent, ConsensusDocEvent, BallotDocEvent, NewRevisionDocEvent, StateDocEvent
@@ -705,3 +707,130 @@ def get_search_cache_key(params):
     key = "doc:document:search:" + hashlib.sha512(json.dumps(kwargs, sort_keys=True)).hexdigest()
     return key
 
+def label_wrap(label, items, joiner=',', max=50):
+    lines = []
+    if not items:
+        return lines
+    line = '%s: %s' % (label, items[0])
+    for item in items[1:]:
+        if len(line)+len(joiner+' ')+len(item) > max:
+            lines.append(line+joiner)
+            line = ' '*(len(label)+len(': ')) + item
+        else:
+            line += joiner+' '+item
+    if line:
+        lines.append(line)
+    return lines
+
+def join_justified(left, right, width=72):
+    count = max(len(left), len(right))
+    left = left + ['']*(count-len(left))
+    right = right + ['']*(count-len(right))
+    lines = []
+    i = 0
+    while True:
+        l = left[i]
+        r = right[i]
+        if len(l)+1+len(r) > width:
+            left = left + ['']
+            right = right[:i] + [''] + right[i:]
+            r = right[i]
+            count += 1
+        lines.append( l + ' ' + r.rjust(width-len(l)-1) )
+        i += 1
+        if i >= count:
+            break
+    return lines
+
+def build_doc_meta_block(doc, path):
+    def add_markup(path, doc, lines):
+        is_hst = doc.is_dochistory()
+        rev = doc.rev
+        if is_hst:
+            doc = doc.doc
+        name = doc.name
+        rfcnum = doc.rfc_number()
+        errata_url = settings.RFC_EDITOR_ERRATA_URL.format(rfc_number=rfcnum) if not is_hst else ""
+        ipr_url = "%s?submit=draft&id=%s" % (urlreverse('ietf.ipr.views.search'), name)
+        for i, line in enumerate(lines):
+            # add draft links
+            line = re.sub(r'\b(draft-[-a-z0-9]+)\b', '<a href="%s/\g<1>">\g<1></a>'%(path, ), line)
+            # add rfcXXXX to RFC links
+            line = re.sub(r' (rfc[0-9]+)\b', ' <a href="%s/\g<1>">\g<1></a>'%(path, ), line)
+            # add XXXX to RFC links
+            line = re.sub(r' ([0-9]{3,5})\b', ' <a href="%s/rfc\g<1>">\g<1></a>'%(path, ), line)
+            # add draft revision links
+            line = re.sub(r' ([0-9]{2})\b', ' <a href="%s/%s-\g<1>">\g<1></a>'%(path, name, ), line)
+            if rfcnum:
+                # add errata link
+                line = re.sub(r'Errata exist', '<a class="text-warning" href="%s">Errata exist</a>'%(errata_url, ), line)
+            if is_hst or not rfcnum:
+                # make current draft rev bold
+                line = re.sub(r'>(%s)<'%rev, '><b>\g<1></b><', line)
+            line = re.sub(r'IPR declarations', '<a class="text-warning" href="%s">IPR declarations</a>'%(ipr_url, ), line)
+            line = line.replace(r'[txt]', '[<a href="%s">txt</a>]' % doc.href())
+            lines[i] = line
+        return lines
+    #
+    now = datetime.datetime.now()
+    draft_state = doc.get_state('draft')
+    block = ''
+    meta = {}
+    if doc.type_id == 'draft':
+        revisions = []
+        ipr = doc.related_ipr()
+        if ipr:
+            meta['ipr'] = [ "IPR declarations" ]
+        if doc.is_rfc() and not doc.is_dochistory():
+            if not doc.name.startswith('rfc'):
+                meta['from'] = [ "%s-%s"%(doc.name, doc.rev) ]
+            meta['errata'] = [ "Errata exist" ] if doc.tags.filter(slug='errata').exists() else []
+            meta['obsoletedby'] = [ alias.document.rfc_number() for alias in doc.related_that('obs') ]
+            meta['obsoletedby'].sort()
+            meta['updatedby'] = [ alias.document.rfc_number() for alias in doc.related_that('updates') ]
+            meta['updatedby'].sort()
+            meta['stdstatus'] = [ doc.std_level.name ]
+        else:
+            dd = doc.doc if doc.is_dochistory() else doc
+            revisions += [ '(%s)%s'%(d.name, ' '*(2-((len(d.name)-1)%3))) for d in dd.replaces() ]
+            revisions += doc.revisions()
+            if doc.is_dochistory() and doc.doc.is_rfc():
+                revisions += [ doc.doc.canonical_name() ]
+            else:
+                revisions += [ d.name for d in doc.replaced_by() ]
+            meta['versions'] = revisions
+            if not doc.is_dochistory and draft_state.slug == 'active' and now > doc.expires:
+                # Active past expiration date
+                meta['active'] = [ 'Document is active' ]
+                meta['state' ] = [ doc.friendly_state() ]
+            intended_std = doc.intended_std_level if doc.intended_std_level else None
+            if intended_std:
+                if intended_std.slug in ['ps', 'ds', 'std']:
+                    meta['stdstatus'] = [ "Standards Track" ]
+                else:
+                    meta['stdstatus'] = [ intended_std.name ]
+    elif doc.type_id == 'charter':
+        meta['versions'] = doc.revisions()
+    #
+    # Add markup to items that needs it.
+    if 'versions' in meta:
+        meta['versions'] = label_wrap('Versions', meta['versions'], joiner="")
+    for label in ['Obsoleted by', 'Updated by', 'From' ]:
+        item = label.replace(' ','').lower()
+        if item in meta and meta[item]:
+            meta[item] = label_wrap(label, meta[item])
+    #
+    left = []
+    right = []
+    #right = [ '[txt]']
+    for item in [ 'from', 'versions', 'obsoletedby', 'updatedby', ]:
+        if item in meta and meta[item]:
+            left += meta[item]
+    for item in ['stdstatus', 'active', 'state', 'ipr', 'errata', ]:
+        if item in meta and meta[item]:
+            right += meta[item]
+    lines = join_justified(left, right)
+    block = '\n'.join(add_markup(path, doc, lines))
+    #
+    return block
+

ietf/doc/views_doc.py

@@ -50,7 +50,7 @@
     can_adopt_draft, get_chartering_type, get_document_content, get_tags_for_stream_id,
     needed_ballot_positions, nice_consensus, prettify_std_name, update_telechat, has_same_ballot,
     get_initial_notify, make_notify_changed_event, crawl_history, default_consensus,
-    add_events_message_info, get_unicode_document_content)
+    add_events_message_info, get_unicode_document_content, build_doc_meta_block)
 from ietf.community.utils import augment_docs_with_tracking_info
 from ietf.group.models import Role
 from ietf.group.utils import can_manage_group_type, can_manage_materials
@@ -67,9 +67,13 @@
 from ietf.review.utils import can_request_review_of_doc, review_requests_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
 
+
 def render_document_top(request, doc, tab, name):
     tabs = []
-    tabs.append(("Document", "document", urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=name)), True, None))
+    tabs.append(("Status", "status", urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=name)), True, None))
+
+    if doc.type_id in ["draft", "charter", ]:
+        tabs.append(("Document", "document", urlreverse("ietf.doc.views_doc.document_html", kwargs=dict(name=name)), True, None))
 
     ballot = doc.latest_event(BallotDocEvent, type="created_ballot")
     if doc.type_id in ("draft","conflrev", "statchg"):
@@ -141,7 +145,7 @@ def document_main(request, name, rev=None):
     # set this after we've found the right doc instance
     group = doc.group
 
-    top = render_document_top(request, doc, "document", name)
+    top = render_document_top(request, doc, "status", name)
 
 
     telechat = doc.latest_event(TelechatDocEvent, type="scheduled_for_telechat")
@@ -597,8 +601,38 @@ def document_main(request, name, rev=None):
                            other_reviews=other_reviews,
                       ))
 
-    raise Http404
+    raise Http404("Document not found: %s" % (name + ("-%s"%rev if rev else "")))
+
+
+def document_html(request, name, rev=None):
+    if name.startswith('rfc0'):
+        name = "rfc" + name[3:].lstrip('0')
+    if name.startswith('review-') and re.search('-\d\d\d\d-\d\d$', name):
+        name = "%s-%s" % (name, rev)
+    docs = Document.objects.filter(docalias__name=name)
+    if not docs.exists():
+        # handle some special cases, like draft-ietf-tsvwg-ieee-802-11
+        name = '%s-%s' % (name, rev)
+        rev=None
+        docs = Document.objects.filter(docalias__name=name)
+    doc = docs.get()
+
+    if not os.path.exists(doc.get_file_name()):
+        raise Http404("Document not found: %s" % doc.get_base_name())
+
+    top = render_document_top(request, doc, "document", name)
+    if not rev and not name.startswith('rfc'):
+        rev = doc.rev
+    if rev:
+        docs = DocHistory.objects.filter(doc=doc, rev=rev)
+        if docs.exists():
+            doc = docs.first()
+        else:
+            doc = doc.fake_history_obj(rev)
+    if doc.type_id in ['draft',]:
+        doc.meta = build_doc_meta_block(doc, settings.HTMLIZER_URL_PREFIX)
 
+    return render(request, "doc/document_html.html", {"doc":doc, "top":top, "navbar_mode":"navbar-static-top",  })
 
 def check_doc_email_aliases():
     pattern = re.compile('^expand-(.*?)(\..*?)?@.*? +(.*)$')

ietf/settings.py

@@ -61,7 +61,7 @@
 TOOLS_SERVER = 'tools.' + IETF_DOMAIN
 TOOLS_SERVER_URL = 'https://' + TOOLS_SERVER
 TOOLS_ID_PDF_URL = TOOLS_SERVER_URL + '/pdf/'
-TOOLS_ID_HTML_URL = TOOLS_SERVER_URL + '/html/'
+TOOLS_ID_HTML_URL = '/doc/html/'
 
 # Override this in the settings_local.py file:
 SERVER_EMAIL = 'Django Server <django-project@' + TOOLS_SERVER + '>'
@@ -524,12 +524,14 @@ def skip_unreadable_post(record):
 DATE_FORMAT = "Y-m-d"
 DATETIME_FORMAT = "Y-m-d H:i T"
 
+
+DRAFT_NAMES_WITH_DOT = "(draft-[a-z-]+-(ion-sig-uni4.0|pilc-2.5g3g|trade-iotp-v1.0-[a-z]+|msword-template-v2.0))"
 URL_REGEXPS = {
     "acronym": r"(?P<acronym>[-a-z0-9]+)",
     "charter": r"(?P<name>charter-[-a-z0-9]+)",
     "date": r"(?P<date>\d{4}-\d{2}-\d{2})",
-    "name": r"(?P<name>[A-Za-z0-9._+-]+)",
-    "rev": r"(?P<rev>[0-9-]+)",
+    "name": r"(?P<name>([A-Za-z0-9_+-]+?|%s))" % DRAFT_NAMES_WITH_DOT,
+    "rev": r"(?P<rev>[0-9-]{2})",
     "owner": r"(?P<owner>[-A-Za-z0-9\'+._]+@[A-Za-z0-9-._]+)",
     "schedule_name": r"(?P<name>[A-Za-z0-9-:_]+)",
 }
@@ -595,9 +597,22 @@ def skip_unreadable_post(record):
     'default': {
         'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
         'LOCATION': '127.0.0.1:11211',
-    }
+        'OPTIONS': {
+            'MAX_ENTRIES': 10000,       # 10,000
+        },
+    },
+    'htmlized': {
+        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+        'LOCATION': '/var/cache/datatracker/htmlized',
+        'OPTIONS': {
+            'MAX_ENTRIES': 100000,      # 100,000
+        },
+    },
 }
 
+HTMLIZER_VERSION = 1
+HTMLIZER_URL_PREFIX = "/doc/html"
+
 IPR_EMAIL_FROM = 'ietf-ipr@ietf.org'
 AUDIO_IMPORT_EMAIL = ['agenda@ietf.org']
 IANA_EVAL_EMAIL = "drafts-eval@icann.org"
@@ -613,6 +628,7 @@ def skip_unreadable_post(record):
 RFC_EDITOR_SYNC_NOTIFICATION_URL = "https://www.rfc-editor.org/parser/parser.php"
 RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml"
 RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml"
+RFC_EDITOR_ERRATA_URL = "https://www.rfc-editor.org/errata_search.php?rfc={rfc_number}&amp;rec_status=0"
 
 # NomCom Tool settings
 ROLODEX_URL = ""
@@ -889,7 +905,15 @@ def skip_unreadable_post(record):
     CACHES = {
          'default': {
              'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
-         }
+         },
+        'htmlized': {
+            'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
+            #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+            'LOCATION': '/var/cache/datatracker/htmlized',
+            'OPTIONS': {
+                'MAX_ENTRIES': 1000,
+            },
+        }
     }
     SESSION_ENGINE = "django.contrib.sessions.backends.db"
 

ietf/settings_testcrawl.py

@@ -19,7 +19,18 @@
 CACHES = {
     'default': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-    }
+        'OPTIONS': {
+            'MAX_ENTRIES': 10000,
+        },
+    },
+    'htmlized': {
+        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
+        #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+        'LOCATION': '/var/cache/datatracker/htmlized',
+        'OPTIONS': {
+            'MAX_ENTRIES': 100000,
+        },
+    },
 }
 
 PASSWORD_HASHERS = ( 'django.contrib.auth.hashers.MD5PasswordHasher', )