Merged in [19262] [19263] [19264] [19265] from rjsparks@nostrum.com:

    Provide replacements for idnits2's internal use of tools.ietf.org:
        - representation of obsoleted RFCs    
        - rfc-status blob
        - minimal per-document .status files
 - Legacy-Id: 19276
Note: SVN reference [19262] has been migrated to Git commit 3283645

Note: SVN reference [19263] has been migrated to Git commit b179143

Note: SVN reference [19264] has been migrated to Git commit d283b96

Note: SVN reference [19265] has been migrated to Git commit 1eb4636

Author: rjsparks

bin/hourly

@@ -85,6 +85,8 @@ mv $TMPFILE9 $DERIVED/1id-index.txt
 mv $TMPFILEA $DERIVED/1id-abstracts.txt
 mv $TMPFILEB $DERIVED/all_id2.txt
 
+$DTDIR/ietf/manage.py generate_idnits2_rfc_status
+$DTDIR/ietf/manage.py generate_idnits2_rfcs_obsoleted
 
 CHARTER=/a/www/ietf-ftp/charter
 wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt

ietf/doc/management/commands/generate_idnits2_rfc_status.py

@@ -0,0 +1,23 @@
+# Copyright The IETF Trust 2021 All Rights Reserved
+
+import os
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from ietf.doc.utils import generate_idnits2_rfc_status
+from ietf.utils.log import log
+
+class Command(BaseCommand):
+    help = ('Generate the rfc_status blob used by idnits2')
+
+    def handle(self, *args, **options):
+        filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfc-status')
+        blob = generate_idnits2_rfc_status()
+        try:
+            bytes = blob.encode('utf-8')
+            with open(filename,'wb') as f:
+                f.write(bytes)
+        except Exception as e:
+            log('failed to write idnits2-rfc-status: '+str(e))
+            raise e

ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py

@@ -0,0 +1,23 @@
+# Copyright The IETF Trust 2021 All Rights Reserved
+
+import os
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from ietf.doc.utils import generate_idnits2_rfcs_obsoleted
+from ietf.utils.log import log
+
+class Command(BaseCommand):
+    help = ('Generate the rfcs-obsoleted file used by idnits2')
+
+    def handle(self, *args, **options):
+        filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfcs-obsoleted')
+        blob = generate_idnits2_rfcs_obsoleted()
+        try:
+            bytes = blob.encode('utf-8')
+            with open(filename,'wb') as f:
+                f.write(bytes)
+        except Exception as e:
+            log('failed to write idnits2-rfcs-obsoleted: '+str(e))
+            raise e

ietf/doc/tests.py

@@ -2377,3 +2377,61 @@ def test_markdown_and_text(self):
         self.assertEqual(r.status_code,200)
         q = PyQuery(r.content)
         self.assertEqual(q('#materials-content .panel-body a').attr['href'],'https://unusual.example')
+
+class Idnits2SupportTests(TestCase):
+
+    def setUp(self):
+        self.derived_dir = self.tempdir('derived')
+        self.saved_derived_dir = settings.DERIVED_DIR
+        settings.DERIVED_DIR = self.derived_dir
+
+    def tearDown(self):
+        settings.DERIVED_DIR = self.saved_derived_dir
+        shutil.rmtree(self.derived_dir)
+
+    def test_obsoleted(self):
+        rfc = WgRfcFactory(alias2__name='rfc1001')
+        WgRfcFactory(alias2__name='rfc1003',relations=[('obs',rfc)])
+        rfc = WgRfcFactory(alias2__name='rfc1005')
+        WgRfcFactory(alias2__name='rfc1007',relations=[('obs',rfc)])
+
+        url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 404)
+        call_command('generate_idnits2_rfcs_obsoleted')
+        url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.content, b'1001 1003\n1005 1007\n')
+
+    def test_rfc_status(self):
+        for slug in ('bcp', 'ds', 'exp', 'hist', 'inf', 'std', 'ps', 'unkn'):
+            WgRfcFactory(std_level_id=slug)
+        url = urlreverse('ietf.doc.views_doc.idnits2_rfc_status')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,404)
+        call_command('generate_idnits2_rfc_status')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,200)
+        blob = unicontent(r).replace('\n','')
+        self.assertEqual(blob[6312-1],'O')
+
+    def test_idnits2_state(self):
+        rfc = WgRfcFactory()
+        url = urlreverse('ietf.doc.views_doc.idnits2_state', kwargs=dict(name=rfc.canonical_name()))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r,'rfcnum')
+
+        draft = WgDraftFactory()
+        url = urlreverse('ietf.doc.views_doc.idnits2_state', kwargs=dict(name=draft.canonical_name()))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertNotContains(r,'rfcnum')
+        self.assertContains(r,'Unknown')
+
+        draft = WgDraftFactory(intended_std_level_id='ps')
+        url = urlreverse('ietf.doc.views_doc.idnits2_state', kwargs=dict(name=draft.canonical_name()))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r,'Proposed')

ietf/doc/urls.py

@@ -69,6 +69,8 @@
     url(r'^html/(?P<name>std[0-9]+?)(\.txt|\.html)?/?$', RedirectView.as_view(url=settings.RFC_EDITOR_INFO_BASE_URL+"%(name)s", permanent=False)), 
     url(r'^html/%(name)s(?:-%(rev)s)?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, views_doc.document_html),
     url(r'^html/(?P<name>[Rr][Ff][Cc] [0-9]+?)(\.txt|\.html)?/?$', views_doc.document_html),
+    url(r'^idnits2-rfcs-obsoleted/?$', views_doc.idnits2_rfcs_obsoleted),
+    url(r'^idnits2-rfc-status/?$', views_doc.idnits2_rfc_status),
 
     url(r'^all/?$', views_search.index_all_drafts),
     url(r'^active/?$', views_search.index_active_drafts),
@@ -78,6 +80,7 @@
 
     url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main),
     url(r'^%(name)s(?:/%(rev)s)?/bibtex/$' % settings.URL_REGEXPS, views_doc.document_bibtex),
+    url(r'^%(name)s(?:/%(rev)s)?/idnits2-state/$' % settings.URL_REGEXPS, views_doc.idnits2_state),
     url(r'^bibxml3/reference.I-D.%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml_ref),
     url(r'^bibxml3/%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml),
     url(r'^%(name)s/history/$' % settings.URL_REGEXPS, views_doc.document_history),

ietf/doc/utils.py

@@ -9,16 +9,19 @@
 import math
 import os
 import re
+import textwrap
 
 from collections import defaultdict
 from urllib.parse import quote
 
 from django.conf import settings
 from django.contrib import messages
 from django.forms import ValidationError
+from django.template.loader import render_to_string
 from django.utils.html import escape
 from django.urls import reverse as urlreverse
 
+
 import debug                            # pyflakes:ignore
 from ietf.community.models import CommunityList
 from ietf.community.utils import docs_tracked_by_community_list
@@ -1232,3 +1235,54 @@ def update_doc_extresources(doc, new_resources, by):
     e.save()
     doc.save_with_history([e])
     return True
+
+def generate_idnits2_rfc_status():
+
+    blob=['N']*10000
+
+    symbols={
+        'ps': 'P',
+        'inf': 'I',
+        'exp': 'E',
+        'ds': 'D',
+        'hist': 'H',
+        'std': 'S',
+        'bcp': 'B',
+        'unkn': 'U',
+    }
+
+    rfcs = Document.objects.filter(type_id='draft',states__slug='rfc',states__type='draft')
+    for rfc in rfcs:
+        offset = int(rfc.rfcnum)-1
+        blob[offset] = symbols[rfc.std_level_id]
+        if rfc.related_that('obs'):
+            blob[offset] = 'O'
+
+    # Workarounds for unusual states in the datatracker
+
+    # Document.get(docalias='rfc6312').rfcnum == 6342 
+    # 6312 was published with the wrong rfc number in it
+    # weird workaround in the datatracker - there are two 
+    # DocAliases starting with rfc - the canonical name code
+    # searches for the lexically highest alias starting with rfc
+    # which is getting lucky.
+    blob[6312 - 1] = 'O'
+
+    # RFC200 is an old RFC List by Number
+    blob[200 -1] = 'O' 
+
+    # End Workarounds
+
+    blob = re.sub('N*$','',''.join(blob))
+    blob = textwrap.fill(blob, width=64)
+
+    return blob
+
+def generate_idnits2_rfcs_obsoleted():
+    obsdict = defaultdict(list)
+    for r in RelatedDocument.objects.filter(relationship_id='obs'):
+        obsdict[int(r.target.document.rfc_number())].append(int(r.source.rfc_number()))
+    for k in obsdict:
+        obsdict[k] = sorted(obsdict[k])
+    return render_to_string('doc/idnits2-rfcs-obsoleted.txt', context={'obsitems':sorted(obsdict.items())})
+

ietf/doc/views_doc.py

@@ -51,6 +51,7 @@
 from django.conf import settings
 from django import forms
 
+
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import ( Document, DocAlias, DocHistory, DocEvent, BallotDocEvent, BallotType,
@@ -79,6 +80,7 @@
 from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
 from ietf.utils import markup_txt, log
+from ietf.utils.draft import Draft
 from ietf.utils.response import permission_denied
 from ietf.utils.text import maybe_split
 
@@ -1714,3 +1716,54 @@ def all_presentations(request, name):
         'in_progress': in_progress,
         'past' : past+recent,
         })
+
+
+def idnits2_rfcs_obsoleted(request):
+    filename = os.path.join(settings.DERIVED_DIR,'idnits2-rfcs-obsoleted')
+    try:
+        with open(filename,'rb') as f:
+            blob = f.read()
+            return HttpResponse(blob,content_type='text/plain;charset=utf-8')
+    except Exception as e:
+        log.log('Failed to read idnits2-rfcs-obsoleted:'+str(e))
+        raise Http404
+
+
+def idnits2_rfc_status(request):
+    filename = os.path.join(settings.DERIVED_DIR,'idnits2-rfc-status')
+    try:
+        with open(filename,'rb') as f:
+            blob = f.read()
+            return HttpResponse(blob,content_type='text/plain;charset=utf-8')
+    except Exception as e:
+        log.log('Failed to read idnits2-rfc-status:'+str(e))
+        raise Http404
+
+
+def idnits2_state(request, name, rev=None):
+    doc = get_object_or_404(Document, docalias__name=name)
+    if doc.type_id!='draft':
+        raise Http404
+    zero_revision = NewRevisionDocEvent.objects.filter(doc=doc,rev='00').first()
+    if zero_revision:
+        doc.created = zero_revision.time
+    else:
+        doc.created = doc.docevent_set.order_by('-time').first().time
+    if doc.std_level:
+        doc.deststatus = doc.std_level.name
+    elif doc.intended_std_level:
+        doc.deststatus = doc.intended_std_level.name
+    else:
+        text = doc.text()
+        if text:
+            parsed_draft = Draft(text=doc.text(), source=name, name_from_source=False)
+            doc.deststatus = parsed_draft.get_status()
+        else:
+            doc.deststatus="Unknown"
+    return render(request, 'doc/idnits2-state.txt', context={'doc':doc}, content_type='text/plain;charset=utf-8')    
+
+
+
+
+
+

ietf/settings.py

@@ -682,6 +682,7 @@ def skip_unreadable_post(record):
 # write anything to this directory -- its content is maintained by ghostlinkd:
 INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/a/ietfdata/doc/draft/archive'
 MEETING_RECORDINGS_DIR = '/a/www/audio'
+DERIVED_DIR = '/a/ietfdata/derived'
 
 DOCUMENT_FORMAT_WHITELIST = ["txt", "ps", "pdf", "xml", "html", ]
 

ietf/templates/doc/idnits2-rfcs-obsoleted.txt

@@ -0,0 +1,2 @@
+{% load ietf_filters %}{% filter linebreaks_lf %}{% for k,l in obsitems %}{{k}} {{l|join:" "}}
+{% endfor %}{% endfilter %}

ietf/templates/doc/idnits2-state.txt

@@ -0,0 +1,6 @@
+{% load ietf_filters %}{% filter linebreaks_lf %}{% comment %}
+{% endcomment %}Doc-created: {{doc.created|date:"Y-m-d"}};datatracker
+Doc-deststatus: {{doc.deststatus}};datatracker
+Doc-rev: {{doc.rev}};datatracker{% if doc.rfcnum %}
+Doc-rfcnum: {{doc.rfcnum}};datatracker{%endif%}
+{% endfilter %}