Added fetch and use of the RFC-Editor's errata.json file in order to set a new 'verified-errata' document tag, and use that to show the link to inline-errata documents only when there is verified errata. Fixes issue ietf-tools#2861.

 - Legacy-Id: 17212

Author: levkowetz

bin/check-copyright

@@ -35,6 +35,7 @@ COPYRIGHT
 """
 from __future__ import print_function
 
+import datetime
 import os
 import sys
 import time
@@ -150,15 +151,18 @@ def get_first_commit(path):
     cmd = 'svn log %s' % path
     if opt_verbose > 1:
         note("Running '%s' ..." % cmd)
-    commit_log = pipe(cmd)
-    commit_log = commit_log.splitlines()
-    commit_log.reverse()
-    for line in commit_log:
-        if re.search(loginfo_format, line):
-            rev, who, when = split_loginfo(line)
-            break
-        else:
-            pass
+    try:
+        commit_log = pipe(cmd)
+        commit_log = commit_log.splitlines()
+        commit_log.reverse()
+        for line in commit_log:
+            if re.search(loginfo_format, line):
+                rev, who, when = split_loginfo(line)
+                break
+            else:
+                pass
+    except OSError:
+        rev, who, when = None, None, datetime.datetime.now()
     return { path: { 'rev': rev, 'who': who, 'date': when.strftime('%Y-%m-%d %H:%M:%S'), }, }
 
 

ietf/bin/rfc-editor-index-updates

@@ -1,8 +1,13 @@
 #!/usr/bin/env python
 
-import os, sys, datetime
+import datetime
+import json
+import os
+import sys
 import traceback
 
+from six.moves.urllib.request import urlopen
+
 # boilerplate
 basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
 sys.path = [ basedir ] + sys.path
@@ -38,15 +43,22 @@ if options.skip_date:
 
 log("Updating document metadata from RFC index from %s" % settings.RFC_EDITOR_INDEX_URL)
 
-response = ietf.sync.rfceditor.fetch_index_xml(settings.RFC_EDITOR_INDEX_URL)
-data = ietf.sync.rfceditor.parse_index(response)
+rfc_index_xml = ietf.sync.rfceditor.fetch_index_xml(settings.RFC_EDITOR_INDEX_URL)
+index_data = ietf.sync.rfceditor.parse_index(rfc_index_xml)
+
+rfc_errata_json = urlopen(settings.RFC_EDITOR_ERRATA_JSON_URL)
+errata_data = json.load(rfc_errata_json)
+
+if len(index_data) < ietf.sync.rfceditor.MIN_INDEX_RESULTS:
+    log("Not enough index entries, only %s" % len(index_data))
+    sys.exit(1)
 
-if len(data) < ietf.sync.rfceditor.MIN_INDEX_RESULTS:
-    log("Not enough results, only %s" % len(data))
+if len(errata_data) < ietf.sync.rfceditor.MIN_ERRATA_RESULTS:
+    log("Not enough errata entries, only %s" % len(errata_data))
     sys.exit(1)
 
 new_rfcs = []
-for changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_index(data, skip_older_than_date=skip_date):
+for changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=skip_date):
     if rfc_published:
         new_rfcs.append(doc)
 

ietf/doc/views_doc.py

@@ -229,7 +229,7 @@ def document_main(request, name, rev=None):
 
             if "txt" in found_types:
                 file_urls.append(("htmlized", settings.TOOLS_ID_HTML_URL + name))
-                if doc.tags.filter(slug="errata"):
+                if doc.tags.filter(slug="verified-errata").exists():
                     file_urls.append(("with errata", settings.RFC_EDITOR_INLINE_ERRATA_URL.format(rfc_number=rfc_number)))
 
             if not found_types:

ietf/name/fixtures/names.json

@@ -9100,6 +9100,16 @@
     "model": "name.doctagname",
     "pk": "sheph-u"
   },
+  {
+    "fields": {
+      "desc": "",
+      "name": "Has verified errata",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.doctagname",
+    "pk": "verified-errata"
+  },
   {
     "fields": {
       "desc": "",
@@ -14216,7 +14226,7 @@
     "fields": {
       "command": "xym",
       "switch": "--version",
-      "time": "2019-12-18T00:13:51.087",
+      "time": "2020-01-09T00:13:22.771",
       "used": true,
       "version": "xym 0.4"
     },
@@ -14227,7 +14237,7 @@
     "fields": {
       "command": "pyang",
       "switch": "--version",
-      "time": "2019-12-18T00:13:52.218",
+      "time": "2020-01-09T00:13:23.631",
       "used": true,
       "version": "pyang 2.1"
     },
@@ -14238,7 +14248,7 @@
     "fields": {
       "command": "yanglint",
       "switch": "--version",
-      "time": "2019-12-18T00:13:52.367",
+      "time": "2020-01-09T00:13:23.912",
       "used": true,
       "version": "yanglint 0.14.80"
     },
@@ -14249,9 +14259,9 @@
     "fields": {
       "command": "xml2rfc",
       "switch": "--version",
-      "time": "2019-12-18T00:13:53.390",
+      "time": "2020-01-09T00:13:24.917",
       "used": true,
-      "version": "xml2rfc 2.36.0"
+      "version": "xml2rfc 2.37.2"
     },
     "model": "utils.versioninfo",
     "pk": 4

ietf/name/migrations/0009_add_verified_errata_to_doctagname.py

@@ -0,0 +1,24 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from django.db import migrations
+
+def forward(apps, schema_editor):
+    DocTagName = apps.get_model('name','DocTagName')
+    DocTagName.objects.get_or_create(slug='verified-errata', name='Has verified errata', desc='', used=True, order=0)
+
+def reverse(apps, schema_editor):
+    DocTagName = apps.get_model('name','DocTagName')
+    DocTagName.objects.filter(slug='verified-errata').delete()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name', '0008_reviewerqueuepolicyname'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse)
+    ]

ietf/settings.py

@@ -734,6 +734,7 @@ def skip_unreadable_post(record):
 #RFC_EDITOR_GROUP_NOTIFICATION_URL = "https://www.rfc-editor.org/notification/group.php"
 RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml"
 RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml"
+RFC_EDITOR_ERRATA_JSON_URL = "https://www.rfc-editor.org/errata.json"
 RFC_EDITOR_ERRATA_URL = "https://www.rfc-editor.org/errata_search.php?rfc={rfc_number}&rec_status=0"
 RFC_EDITOR_INLINE_ERRATA_URL = "https://www.rfc-editor.org/rfc/inline-errata/rfc{rfc_number}.html"
 

ietf/sync/rfceditor.py

@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2012-2019, All Rights Reserved
+# Copyright The IETF Trust 2012-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -32,8 +32,9 @@
 #INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml"
 #POST_APPROVED_DRAFT_URL = "https://www.rfc-editor.org/sdev/jsonexp/jsonparser.php"
 
+MIN_ERRATA_RESULTS = 5000
+MIN_INDEX_RESULTS = 8000
 MIN_QUEUE_RESULTS = 10
-MIN_INDEX_RESULTS = 5000
 
 def get_child_text(parent_node, tag_name):
     text = []
@@ -330,11 +331,18 @@ def extract_doc_list(parentNode, tagName):
     return data
 
 
-def update_docs_from_rfc_index(data, skip_older_than_date=None):
+def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=None):
     """Given parsed data from the RFC Editor index, update the documents
     in the database. Yields a list of change descriptions for each
     document, if any."""
 
+    errata = {}
+    for item in errata_data:
+        name = item['doc-id']
+        if not name in errata:
+            errata[name] = []
+        errata[name].append(item)
+
     std_level_mapping = {
         "Standard": StdLevelName.objects.get(slug="std"),
         "Internet Standard": StdLevelName.objects.get(slug="std"),
@@ -356,12 +364,13 @@ def update_docs_from_rfc_index(data, skip_older_than_date=None):
     }
 
     tag_has_errata = DocTagName.objects.get(slug='errata')
+    tag_has_verified_errata = DocTagName.objects.get(slug='verified-errata')
     relationship_obsoletes = DocRelationshipName.objects.get(slug="obs")
     relationship_updates = DocRelationshipName.objects.get(slug="updates")
 
     system = Person.objects.get(name="(System)")
 
-    for rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract in data:
+    for rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract in index_data:
 
         if skip_older_than_date and rfc_published_date < skip_older_than_date:
             # speed up the process by skipping old entries
@@ -493,13 +502,20 @@ def parse_relation_list(l):
                     changes.append("created alias %s" % prettify_std_name(a))
 
         if has_errata:
-            if not doc.tags.filter(pk=tag_has_errata.pk):
+            if not doc.tags.filter(pk=tag_has_errata.pk).exists():
                 doc.tags.add(tag_has_errata)
                 changes.append("added Errata tag")
+            has_verified_errata = name.upper() in errata and any([ e['errata_status_code']=='Verified' for e in errata[name.upper()] ])
+            if has_verified_errata and not doc.tags.filter(pk=tag_has_verified_errata.pk).exists():
+                doc.tags.add(tag_has_verified_errata)
+                changes.append("added Verified Errata tag")
         else:
             if doc.tags.filter(pk=tag_has_errata.pk):
                 doc.tags.remove(tag_has_errata)
                 changes.append("removed Errata tag")
+            if doc.tags.filter(pk=tag_has_verified_errata.pk):
+                doc.tags.remove(tag_has_verified_errata)
+                changes.append("removed Verified Errata tag")
 
         if changes:
             events.append(DocEvent.objects.create(

ietf/sync/tests.py

@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2012-2019, All Rights Reserved
+# Copyright The IETF Trust 2012-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -315,6 +315,22 @@ def test_rfc_index(self):
                        area=doc.group.parent.acronym,
                        group=doc.group.acronym)
 
+        errata = [{
+                "errata_id":1,
+                "doc-id":"RFC123",
+                "errata_status_code":"Verified",
+                "errata_type_code":"Editorial",
+                "section": "4.1",
+                "orig_text":"   S: 220-smtp.example.com ESMTP Server",
+                "correct_text":"   S: 220 smtp.example.com ESMTP Server",
+                "notes":"There are 3 instances of this (one on p. 7 and two on p. 8). \n",
+                "submit_date":"2007-07-19",
+                "submitter_name":"Rob Siemborski",
+                "verifier_id":99,
+                "verifier_name":None,
+                "update_date":"2019-09-10 09:09:03"},
+        ]
+
         data = rfceditor.parse_index(io.StringIO(t))
         self.assertEqual(len(data), 1)
 
@@ -339,7 +355,7 @@ def test_rfc_index(self):
         self.write_draft_file(draft_filename, 5000)
 
         changes = []
-        for cs, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30)):
+        for cs, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)):
             changes.append(cs)
 
         doc = Document.objects.get(name=doc.name)
@@ -364,7 +380,7 @@ def test_rfc_index(self):
         self.assertTrue(os.path.exists(os.path.join(self.archive_dir, draft_filename)))
 
         # make sure we can apply it again with no changes
-        changed = list(rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30)))
+        changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)))
         self.assertEqual(len(changed), 0)