Merged in [18157] from jennifer@painless-security.com:

    Store Auth48 URL as DocumentURL and display in RFC-Editor state. Migrates old data when possible. Alternative to 17563. Fixes ietf-tools#2722.
 - Legacy-Id: 18166
Note: SVN reference [18157] has been migrated to Git commit fff927b

Author: levkowetz

ietf/doc/migrations/0032_populate_auth48_urls.py

@@ -0,0 +1,67 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+from django.db.models import OuterRef, Subquery
+
+from re import match
+
+
+def forward(apps, schema_editor):
+    """Add DocumentURLs for docs in the Auth48 state
+
+    Checks the latest StateDocEvent; if it is in the auth48 state and the
+    event desc has an AUTH48 link, creates an auth48 DocumentURL for that doc.
+    """
+    Document = apps.get_model('doc', 'Document')
+    StateDocEvent = apps.get_model('doc', 'StateDocEvent')
+    DocumentURL = apps.get_model('doc', 'DocumentURL')
+
+    # Regex - extracts auth48 URL as first match group
+    pattern = r'RFC Editor state changed to <a href="(.*)"><b>AUTH48.*</b></a>.*'
+
+    # To avoid 100k queries, set up a subquery to find the latest StateDocEvent for each doc...
+    latest_events = StateDocEvent.objects.filter(doc=OuterRef('pk')).order_by('-time', '-id')
+    # ... then annotate the doc list with that and select only those in the auth48 state...
+    auth48_docs = Document.objects.annotate(
+        current_state_slug=Subquery(latest_events.values('state__slug')[:1])
+    ).filter(current_state_slug='auth48')
+    # ... and add an auth48 DocumentURL if one is found.
+    for doc in auth48_docs:
+        # Retrieve the full StateDocEvent. Results in a query per doc, but
+        # only for the few few in the auth48 state.
+        sde = StateDocEvent.objects.filter(doc=doc).order_by('-time', '-id').first()
+        urlmatch = match(pattern, sde.desc)  # Auth48 URL is usually in the event desc
+        if urlmatch is not None:
+            DocumentURL.objects.create(doc=doc, tag_id='auth48', url=urlmatch[1])
+
+    # Validate the migration using a different approach to find auth48 docs.
+    # This is slower than above, but still avoids querying for every Document.
+    auth48_events = StateDocEvent.objects.filter(state__slug='auth48')
+    for a48_event in auth48_events:
+        doc = a48_event.doc
+        latest_sde = StateDocEvent.objects.filter(doc=doc).order_by('-time', '-id').first()
+        if latest_sde.state and latest_sde.state.slug == 'auth48' and match(pattern, latest_sde.desc) is not None:
+            # Currently in the auth48 state with a URL
+            assert doc.documenturl_set.filter(tag_id='auth48').count() == 1
+        else:
+            # Either no longer in auth48 state or had no URL
+            assert doc.documenturl_set.filter(tag_id='auth48').count() == 0
+
+
+def reverse(apps, schema_editor):
+    """Remove any auth48 DocumentURLs - these did not exist before"""
+    DocumentURL = apps.get_model('doc', 'DocumentURL')
+    DocumentURL.objects.filter(tag_id='auth48').delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('doc', '0031_set_state_for_charters_of_replaced_groups'),
+        ('name', '0012_add_auth48_docurltagname'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]

ietf/doc/models.py

@@ -916,6 +916,7 @@ def is_dochistory(self):
     def related_ipr(self):
         return self.doc.related_ipr()
 
+    @property
     def documenturl_set(self):
         return self.doc.documenturl_set
 

ietf/doc/tests.py

@@ -780,6 +780,44 @@ def test_document_primary_and_history_views(self):
             self.assertEqual(r.status_code, 200)
             self.assertContains(r, "%s-00"%docname)
 
+    def test_rfcqueue_auth48_views(self):
+        """Test view handling of RFC editor queue auth48 state"""
+        def _change_state(doc, state):
+            event = StateDocEventFactory(doc=doc, state=state)
+            doc.set_state(event.state)
+            doc.save_with_history([event])
+
+        draft = IndividualDraftFactory()
+
+        # Put in an rfceditor state other than auth48
+        for state in [('draft-iesg', 'rfcqueue'), ('draft-rfceditor', 'rfc-edit')]:
+            _change_state(draft, state)
+        r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
+        self.assertEqual(r.status_code, 200)
+        self.assertNotContains(r, 'Auth48 status')
+
+        # Put in auth48 state without a URL
+        _change_state(draft, ('draft-rfceditor', 'auth48'))
+        r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
+        self.assertEqual(r.status_code, 200)
+        self.assertNotContains(r, 'Auth48 status')
+
+        # Now add a URL
+        documenturl = draft.documenturl_set.create(tag_id='auth48', 
+                                                   url='http://rfceditor.example.com/auth48-url')
+        r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, 'Auth48 status')
+        self.assertContains(r, documenturl.url)
+
+        # Put in auth48-done state and delete auth48 DocumentURL
+        draft.documenturl_set.filter(tag_id='auth48').delete()
+        _change_state(draft, ('draft-rfceditor', 'auth48-done'))
+        r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
+        self.assertEqual(r.status_code, 200)
+        self.assertNotContains(r, 'Auth48 status')
+
+
 class DocTestCase(TestCase):
     def test_document_charter(self):
         CharterFactory(name='charter-ietf-mars')

ietf/doc/views_doc.py

@@ -420,6 +420,15 @@ def document_main(request, name, rev=None):
         exp_comment = doc.latest_event(IanaExpertDocEvent,type="comment")
         iana_experts_comment = exp_comment and exp_comment.desc
 
+        # See if we should show an Auth48 URL
+        auth48_url = None  # stays None unless we are in the auth48 state
+        if doc.get_state_slug('draft-rfceditor') == 'auth48':
+            document_url = doc.documenturl_set.filter(tag_id='auth48').first()
+            auth48_url = document_url.url if document_url else ''
+
+        # Do not show the Auth48 URL in the "Additional URLs" section
+        additional_urls = doc.documenturl_set.exclude(tag_id='auth48')
+
         return render(request, "doc/document_draft.html",
                                   dict(doc=doc,
                                        group=group,
@@ -469,6 +478,7 @@ def document_main(request, name, rev=None):
                                        has_errata=doc.tags.filter(slug="errata"),
                                        published=published,
                                        file_urls=file_urls,
+                                       additional_urls=additional_urls,
                                        stream_state_type_slug=stream_state_type_slug,
                                        stream_state=stream_state,
                                        stream_tags=stream_tags,
@@ -477,6 +487,7 @@ def document_main(request, name, rev=None):
                                        iesg_state=iesg_state,
                                        iesg_state_summary=iesg_state_summary,
                                        rfc_editor_state=doc.get_state("draft-rfceditor"),
+                                       rfc_editor_auth48_url=auth48_url,
                                        iana_review_state=doc.get_state("draft-iana-review"),
                                        iana_action_state=doc.get_state("draft-iana-action"),
                                        iana_experts_state=doc.get_state("draft-iana-experts"),

ietf/name/fixtures/names.json

@@ -9510,6 +9510,16 @@
     "model": "name.doctypename",
     "pk": "statchg"
   },
+  {
+    "fields": {
+      "desc": "",
+      "name": "RFC Editor Auth48 status",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.docurltagname",
+    "pk": "auth48"
+  },
   {
     "fields": {
       "desc": "",
@@ -14584,7 +14594,7 @@
     "fields": {
       "command": "xym",
       "switch": "--version",
-      "time": "2020-07-11T00:12:44.590",
+      "time": "2020-05-29T00:13:35.959",
       "used": true,
       "version": "xym 0.4.8"
     },
@@ -14595,7 +14605,7 @@
     "fields": {
       "command": "pyang",
       "switch": "--version",
-      "time": "2020-07-11T00:12:46.120",
+      "time": "2020-05-29T00:13:38.724",
       "used": true,
       "version": "pyang 2.2.1"
     },
@@ -14606,7 +14616,7 @@
     "fields": {
       "command": "yanglint",
       "switch": "--version",
-      "time": "2020-07-11T00:12:46.381",
+      "time": "2020-05-29T00:13:39.026",
       "used": true,
       "version": "yanglint SO 1.6.7"
     },
@@ -14617,7 +14627,7 @@
     "fields": {
       "command": "xml2rfc",
       "switch": "--version",
-      "time": "2020-07-11T00:12:48.052",
+      "time": "2020-05-29T00:13:40.790",
       "used": true,
       "version": "xml2rfc 2.46.0"
     },

ietf/name/migrations/0012_add_auth48_docurltagname.py

@@ -0,0 +1,28 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 2.0.13 on 2020-06-02 10:13
+
+from django.db import migrations
+
+def forward(apps, schema_editor):
+    DocUrlTagName = apps.get_model('name', 'DocUrlTagName')
+    DocUrlTagName.objects.create(
+        slug='auth48',
+        name='RFC Editor Auth48 status',
+        used=True,
+    )
+
+def reverse(apps, schema_editor):
+    DocUrlTagName = apps.get_model('name', 'DocUrlTagName')
+    auth48_tag = DocUrlTagName.objects.get(slug='auth48')
+    auth48_tag.delete()
+
+class Migration(migrations.Migration):
+    """Add DocUrlTagName entry for RFC Ed Auth48 URL"""
+    dependencies = [
+        ('name', '0011_constraintname_editor_label'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]

ietf/sync/rfceditor.py

@@ -199,7 +199,14 @@ def update_drafts_from_queue(drafts):
             if auth48:
                 e.desc = re.sub(r"(<b>.*</b>)", "<a href=\"%s\">\\1</a>" % auth48, e.desc)
                 e.save()
-
+                # Create or update the auth48 URL whether or not this is a state expected to have one.
+                d.documenturl_set.update_or_create(
+                    tag_id='auth48',  # look up existing based on this field
+                    defaults=dict(url=auth48)  # create or update with this field
+                )
+            else:
+                # Remove any existing auth48 URL when an update does not have one.
+                d.documenturl_set.filter(tag_id='auth48').delete()
             if e:
                 events.append(e)
 

ietf/sync/tests.py

@@ -381,17 +381,15 @@ def test_rfc_index(self):
         changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)))
         self.assertEqual(len(changed), 0)
 
-
-    def test_rfc_queue(self):
-        draft = WgDraftFactory(states=[('draft-iesg','ann')])
-
+    def _generate_rfc_queue_xml(self, draft, state, auth48_url=None):
+        """Generate an RFC queue xml string for a draft"""
         t = '''<rfc-editor-queue xmlns="http://www.rfc-editor.org/rfc-editor-queue">
 <section name="IETF STREAM: WORKING GROUP STANDARDS TRACK">
 <entry xml:id="%(name)s">
 <draft>%(name)s-%(rev)s.txt</draft>
 <date-received>2010-09-08</date-received>
-<state>EDIT*R*A(1G)</state>
-<auth48-url>http://www.rfc-editor.org/auth48/rfc1234</auth48-url>
+<state>%(state)s</state>
+<auth48-url>%(auth48_url)s</auth48-url>
 <normRef>
 <ref-name>%(ref)s</ref-name>
 <ref-state>IN-QUEUE</ref-state>
@@ -408,26 +406,24 @@ def test_rfc_queue(self):
                               rev=draft.rev,
                               title=draft.title,
                               group=draft.group.name,
-                              ref="draft-ietf-test")
+                              ref="draft-ietf-test",
+                              state=state,
+                              auth48_url=(auth48_url or ''))
+        t = t.replace('<auth48-url></auth48-url>\n', '')  # strip empty auth48-url tags
+        return t
+
+    def test_rfc_queue(self):
+        draft = WgDraftFactory(states=[('draft-iesg','ann')])
+        expected_auth48_url = "http://www.rfc-editor.org/auth48/rfc1234"
+        t = self._generate_rfc_queue_xml(draft,
+                                         state='EDIT*R*A(1G)',
+                                         auth48_url=expected_auth48_url)
 
         drafts, warnings = rfceditor.parse_queue(io.StringIO(t))
+        # rfceditor.parse_queue() is tested independently; just sanity check here
         self.assertEqual(len(drafts), 1)
         self.assertEqual(len(warnings), 0)
 
-        # Test with TI state introduced 11 Sep 2019
-        t = t.replace("<state>EDIT*R*A(1G)</state>", "<state>TI</state>")
-        __, warnings = rfceditor.parse_queue(io.StringIO(t))
-        self.assertEqual(len(warnings), 0)
-
-        draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs = drafts[0]
-
-        # currently, we only check what we actually use
-        self.assertEqual(draft_name, draft.name)
-        self.assertEqual(state, "EDIT")
-        self.assertEqual(set(tags), set(["iana", "ref"]))
-        self.assertEqual(auth48, "http://www.rfc-editor.org/auth48/rfc1234")
-
-
         mailbox_before = len(outbox)
 
         changed, warnings = rfceditor.update_drafts_from_queue(drafts)
@@ -450,6 +446,83 @@ def test_rfc_queue(self):
         self.assertEqual(len(changed), 0)
         self.assertEqual(len(warnings), 0)
 
+    def test_rfceditor_parse_queue(self):
+        """Test that rfceditor.parse_queue() behaves as expected.
+
+        Currently does a limited test - old comment was 
+        "currently, we only check what we actually use".
+        """
+        draft = WgDraftFactory(states=[('draft-iesg','ann')])
+        t = self._generate_rfc_queue_xml(draft,
+                                         state='EDIT*R*A(1G)',
+                                         auth48_url="http://www.rfc-editor.org/auth48/rfc1234")
+
+        drafts, warnings = rfceditor.parse_queue(io.StringIO(t))
+        self.assertEqual(len(drafts), 1)
+        self.assertEqual(len(warnings), 0)
+
+        draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs = drafts[0]
+        self.assertEqual(draft_name, draft.name)
+        self.assertEqual(state, "EDIT")
+        self.assertEqual(set(tags), set(["iana", "ref"]))
+        self.assertEqual(auth48, "http://www.rfc-editor.org/auth48/rfc1234")
+
+    def test_rfceditor_parse_queue_TI_state(self):
+        # Test with TI state introduced 11 Sep 2019
+        draft = WgDraftFactory(states=[('draft-iesg','ann')])
+        t = self._generate_rfc_queue_xml(draft,
+                                         state='TI',
+                                         auth48_url="http://www.rfc-editor.org/auth48/rfc1234")
+        __, warnings = rfceditor.parse_queue(io.StringIO(t))
+        self.assertEqual(len(warnings), 0)
+
+    def _generate_rfceditor_update(self, draft, state, tags=None, auth48_url=None):
+        """Helper to generate fake output from rfceditor.parse_queue()"""
+        return [[
+            draft.name, # draft_name
+            '2020-06-03',  # date_received
+            state,
+            tags or [],
+            '1',  # missref_generation
+            'ietf',  # stream
+            auth48_url or '',
+            '',  # cluster
+            ['draft-ietf-test'],  # refs
+        ]]
+
+    def test_update_draft_auth48_url(self):
+        """Test that auth48 URLs are handled correctly."""
+        draft = WgDraftFactory(states=[('draft-iesg','ann')])
+
+        # Step 1 setup: update to a state with no auth48 URL
+        changed, warnings = rfceditor.update_drafts_from_queue(
+            self._generate_rfceditor_update(draft, state='EDIT')
+        )
+        self.assertEqual(len(changed), 1)
+        self.assertEqual(len(warnings), 0)
+        auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first()
+        self.assertIsNone(auth48_docurl)
+
+        # Step 2: update to auth48 state with auth48 URL
+        changed, warnings = rfceditor.update_drafts_from_queue(
+            self._generate_rfceditor_update(draft, state='AUTH48', auth48_url='http://www.rfc-editor.org/rfc1234')
+        )
+        self.assertEqual(len(changed), 1)
+        self.assertEqual(len(warnings), 0)
+        auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first()
+        self.assertIsNotNone(auth48_docurl)
+        self.assertEqual(auth48_docurl.url, 'http://www.rfc-editor.org/rfc1234')
+
+        # Step 3: update to auth48-done state without auth48 URL
+        changed, warnings = rfceditor.update_drafts_from_queue(
+            self._generate_rfceditor_update(draft, state='AUTH48-DONE')
+        )
+        self.assertEqual(len(changed), 1)
+        self.assertEqual(len(warnings), 0)
+        auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first()
+        self.assertIsNone(auth48_docurl)
+
+
 class DiscrepanciesTests(TestCase):
     def test_discrepancies(self):