Merged in [13181] from housley@vigilsec.com:

This completes a good chunk of the downref registry work requested in ticket
ietf-tools#2069. The registry has been imported to the database and can be shown, and
entries can be added to the registry.  Addresses issue ietf-tools#2069.
 - Legacy-Id: 13190
Note: SVN reference [13181] has been migrated to Git commit f012563

Author: levkowetz

ietf/doc/forms.py

@@ -1,7 +1,13 @@
-import datetime
+# Copyright The IETF Trust 2017, All Rights Reserved
+
+from __future__ import unicode_literals
 
+import datetime
+import debug #pyflakes:ignore
 from django import forms
 
+from ietf.doc.fields import SearchableDocAliasesField, SearchableDocAliasField
+from ietf.doc.models import RelatedDocument
 from ietf.iesg.models import TelechatDate
 from ietf.iesg.utils import telechat_page_count
 
@@ -45,3 +51,64 @@ class NotifyForm(forms.Form):
     def clean_notify(self):
         addrspecs = [x.strip() for x in self.cleaned_data["notify"].split(',')]
         return ', '.join(addrspecs)
+
+IESG_APPROVED_STATE_LIST = ("ann", "rfcqueue", "pub")
+
+class AddDownrefForm(forms.Form):
+    rfc = SearchableDocAliasField(
+                label="Referenced RFC",
+                help_text="The RFC that is approved for downref",
+                required=True)
+    drafts = SearchableDocAliasesField(
+                label="Internet-Drafts that makes the reference",
+                help_text="The drafts that approve the downref in their Last Call",
+                required=True)
+
+    def clean_rfc(self):
+        if 'rfc' not in self.cleaned_data:
+            raise forms.ValidationError("Please provide a referenced RFC and a referencing Internet-Draft")
+
+        rfc = self.cleaned_data['rfc']
+        if not rfc.document.is_rfc():
+            raise forms.ValidationError("Cannot find the RFC: " + rfc.name)
+        return rfc
+
+    def clean_drafts(self):
+        if 'drafts' not in self.cleaned_data:
+            raise forms.ValidationError("Please provide a referenced RFC and a referencing Internet-Draft")
+
+        v_err_names = []
+        drafts = self.cleaned_data['drafts']
+        for da in drafts:
+            state = da.document.get_state("draft-iesg")
+            if not state or state.slug not in IESG_APPROVED_STATE_LIST:
+                v_err_names.append(da.name)
+        if v_err_names:
+            raise forms.ValidationError("Draft is not yet approved: " + ", ".join(v_err_names))
+        return drafts
+
+    def clean(self):
+        if 'rfc' not in self.cleaned_data or 'drafts' not in self.cleaned_data:
+            raise forms.ValidationError("Please provide a referenced RFC and a referencing Internet-Draft")
+
+        v_err_pairs = []
+        rfc = self.cleaned_data['rfc']
+        drafts = self.cleaned_data['drafts']
+        for da in drafts:
+            if RelatedDocument.objects.filter(source=da.document, target=rfc, relationship_id='downref-approval'):
+                v_err_pairs.append(da.name + " --> RFC " + rfc.document.rfc_number())
+        if v_err_pairs:
+            raise forms.ValidationError("Downref is already in the registry: " + ", ".join(v_err_pairs))
+
+        if 'save_downref_anyway' not in self.data:
+        # this check is skipped if the save_downref_anyway button is used
+            v_err_refnorm = ""
+            for da in drafts:
+                if not RelatedDocument.objects.filter(source=da.document, target=rfc, relationship_id='refnorm'):
+                    if v_err_refnorm:
+                        v_err_refnorm = v_err_refnorm + " or " + da.name
+                    else:
+                        v_err_refnorm = da.name
+            if v_err_refnorm:
+                v_err_refnorm_prefix = "There does not seem to be a normative reference to RFC " + rfc.document.rfc_number() + " by "
+                raise forms.ValidationError(v_err_refnorm_prefix  + v_err_refnorm)

ietf/doc/migrations/0026_add_downrefappr_from_wiki.py

@@ -0,0 +1,129 @@
+# Copyright The IETF Trust 2017, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+downref_registry_from_wiki = [
+    ['rfc952', 'draft-hollenbeck-rfc4931bis'],
+    ['rfc952', 'draft-hollenbeck-rfc4932bis'],
+    ['rfc1094','draft-ietf-nfsv4-nfsdirect'],
+    ['rfc1321','rfc3967'],
+    ['rfc1813','draft-ietf-nfsv4-nfsdirect'],
+    ['rfc1951','draft-ietf-lemonade-compress'],
+    ['rfc1952','draft-sweet-rfc2911bis'],
+    ['rfc1977','draft-sweet-rfc2911bis'],
+    ['rfc2104','rfc3967'],
+    ['rfc2144','draft-ietf-secsh-newmodes'],
+    ['rfc2315','draft-eastlake-additional-xmlsec-uris'],
+    ['rfc2330','draft-ietf-ippm-metrictest'],
+    ['rfc2412','draft-ietf-cat-kerberos-pk-init'],
+    ['rfc2648','draft-ietf-simple-xcap-diff'],
+    ['rfc2683','draft-ietf-qresync-rfc5162bis'],
+    ['rfc2702','draft-ietf-isis-admin-tags'],
+    ['rfc2781','draft-ietf-appsawg-xml-mediatypes'],
+    ['rfc2818','draft-dusseault-caldav'],
+    ['rfc2898','draft-turner-asymmetrickeyformat-algs'],
+    ['rfc2966','draft-ietf-isis-admin-tags'],
+    ['rfc2985','rfc5750'],
+    ['rfc2986','rfc6487'],
+    ['rfc3032','draft-ietf-pals-rfc4447bis'],
+    ['rfc3174','draft-harris-ssh-rsa-kex'],
+    ['rfc3196','draft-sweet-rfc2911bis'],
+    ['rfc3217','draft-ietf-smime-cms-rsa-kem'],
+    ['rfc3272','draft-ietf-mpls-cosfield-def'],
+    ['rfc3280','rfc3852'],
+    ['rfc3281','rfc3852'],
+    ['rfc3394','draft-ietf-smime-cms-rsa-kem'],
+    ['rfc3447','draft-ietf-cat-kerberos-pk-init'],
+    ['rfc3469','draft-ietf-mpls-cosfield-def'],
+    ['rfc3548','draft-ietf-dnsext-dnssec-records'],
+    ['rfc3564','draft-ietf-mpls-cosfield-def'],
+    ['rfc3567','draft-ietf-pce-disco-proto-isis'],
+    ['rfc3610','rfc4309'],
+    ['rfc3843','rfc5953'],
+    ['rfc3579','draft-ietf-radext-rfc4590bis'],
+    ['rfc3618','draft-ietf-mboned-msdp-deploy'],
+    ['rfc3713','draft-kato-ipsec-ciph-camellia'],
+    ['rfc3784','draft-ietf-isis-admin-tags'],
+    ['rfc3985','draft-ietf-mpls-cosfield-def'],
+    ['rfc4050','draft-eastlake-additional-xmlsec-uris'],
+    ['rfc4082','draft-ietf-msec-srtp-tesla'],
+    ['rfc4226','draft-ietf-keyprov-pskc'],
+    ['rfc4269','draft-eastlake-additional-xmlsec-uris'],
+    ['rfc4291','draft-hollenbeck-rfc4932bis'],
+    ['rfc4347','rfc5953'],
+    ['rfc4357','draft-ietf-pkix-gost-cppk'],
+    ['rfc4366','rfc5953'],
+    ['rfc4492','draft-ietf-tls-chacha20-poly1305'],
+    ['rfc4493','draft-songlee-aes-cmac-96'],
+    ['rfc4627','draft-ietf-mediactrl-ivr-control-package'],
+    ['rfc4753','draft-ietf-ipsec-ike-auth-ecdsa'],
+    ['rfc4949','draft-ietf-oauth-v2'],
+    ['rfc5036','draft-ietf-pals-rfc4447bis'],
+    ['rfc5246','rfc5953'],
+    ['rfc5280','rfc5953'],
+    ['rfc5322','draft-hollenbeck-rfc4933bis'],
+    ['rfc5410','draft-arkko-mikey-iana'],
+    ['rfc5489','draft-ietf-tls-chacha20-poly1305'],
+    ['rfc5598','draft-ietf-dkim-mailinglists'],
+    ['rfc5649','draft-turner-asymmetrickeyformat-algs'],
+    ['rfc5753','draft-turner-cms-symmetrickeypackage-algs'],
+    ['rfc5781','draft-ietf-sidr-res-certs'],
+    ['rfc5869','draft-ietf-trill-channel-tunnel'],
+    ['rfc5890','draft-ietf-dkim-rfc4871bis'],
+    ['rfc5911','draft-turner-asymmetrickeyformat'],
+    ['rfc5912','draft-ietf-pkix-authorityclearanceconstraints'],
+    ['rfc5952','rfc5953'],
+    ['rfc6043','draft-arkko-mikey-iana'],
+    ['rfc6090','draft-turner-akf-algs-update'],
+    ['rfc6151','draft-ietf-netmod-system-mgmt'],
+    ['rfc6234','draft-schaad-pkix-rfc2875-bis'],
+    ['rfc6386','draft-ietf-rtcweb-video'],
+    ['rfc6480','rfc6485'],
+    ['rfc6480','rfc6489'],
+    ['rfc6480','rfc6491'],
+    ['rfc6480','rfc7935'],
+    ['rfc6707','draft-ietf-cdni-metadata'],
+    ['rfc6839','draft-ietf-appsawg-xml-mediatypes'],
+    ['rfc7251','rfc7252'],
+    ['rfc7358','draft-ietf-pals-rfc4447bis'],
+    ['rfc7539','draft-ietf-tls-chacha20-poly1305'],
+    ['rfc7612','draft-sweet-rfc2911bis'],
+    ['rfc7748','draft-ietf-jose-cfrg-curves'],
+    ['rfc8032','draft-ietf-jose-cfrg-curves'] ]
+
+
+def addDownrefRelationships(apps,schema_editor):
+    DocAlias = apps.get_model('doc','DocAlias')
+    RelatedDocument = apps.get_model('doc','RelatedDocument')
+
+    for [fn2, fn1] in downref_registry_from_wiki:
+        da1 = DocAlias.objects.get(name=fn1)
+        da2 = DocAlias.objects.get(name=fn2)
+        RelatedDocument.objects.create(source=da1.document,
+             target=da2, relationship_id='downref-approval')
+
+
+def removeDownrefRelationships(apps,schema_editor):
+    DocAlias = apps.get_model('doc','DocAlias')
+    RelatedDocument = apps.get_model('doc','RelatedDocument')
+
+    for [fn2, fn1] in downref_registry_from_wiki:
+        da1 = DocAlias.objects.get(name=fn1)
+        da2 = DocAlias.objects.get(name=fn2)
+        RelatedDocument.objects.filter(source=da1.document,
+             target=da2, relationship_id='downref-approval').delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name', '0019_add_docrelationshipname_downref_approval'),
+        ('doc', '0025_auto_20170307_0146'),
+    ]
+
+    operations = [
+        migrations.RunPython(addDownrefRelationships,removeDownrefRelationships)
+    ]

ietf/doc/models.py

@@ -888,6 +888,9 @@ class DocReminder(models.Model):
     ("requested_review", "Requested review"),
     ("assigned_review_request", "Assigned review request"),
     ("closed_review_request", "Closed review request"),
+
+    # downref
+    ("downref_approved", "Downref approved"),
     ]
 
 class DocEvent(models.Model):

ietf/doc/tests_downref.py

@@ -0,0 +1,109 @@
+# Copyright The IETF Trust 2017, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from django.core.urlresolvers import reverse as urlreverse
+
+import debug    # pyflakes:ignore
+
+from ietf.doc.models import Document, DocAlias, RelatedDocument, State
+from ietf.utils.test_utils import TestCase
+from ietf.utils.test_data  import make_test_data, make_downref_test_data
+from ietf.utils.test_utils import login_testing_unauthorized, unicontent
+
+class Downref(TestCase):
+    def test_downref_registry(self):
+        url = urlreverse('ietf.doc.views_downref.downref_registry')
+
+        # normal - get the table without the "Add downref" button
+        self.client.login(username="plain", password="plain+password")
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        content = unicontent(r)
+        self.assertTrue('<h1>Downref registry</h1>' in content)
+        self.assertFalse('Add downref' in content)
+
+        # secretariat - get the table with the "Add downref" button
+        self.client.login(username='secretary', password='secretary+password')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        content = unicontent(r)
+        self.assertTrue('<h1>Downref registry</h1>' in content)
+        self.assertTrue('Add downref' in content)
+
+        # area director - get the table with the "Add downref" button
+        self.client.login(username='ad', password='ad+password')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        content = unicontent(r)
+        self.assertTrue('<h1>Downref registry</h1>' in content)
+        self.assertTrue('Add downref' in content)
+
+    def test_downref_registry_add(self):
+        url = urlreverse('ietf.doc.views_downref.downref_registry_add')
+        login_testing_unauthorized(self, "plain", url)
+
+        # secretariat - get the form to add entries to the registry
+        self.client.login(username='secretary', password='secretary+password')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        content = unicontent(r)
+        self.assertTrue('<h1>Add entry to the downref registry</h1>' in content)
+        self.assertTrue('Save downref' in content)
+
+        # area director - get the form to add entries to the registry
+        self.client.login(username='ad', password='ad+password')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        content = unicontent(r)
+        self.assertTrue('<h1>Add entry to the downref registry</h1>' in content)
+        self.assertTrue('Save downref' in content)
+
+        # error - already in the downref registry
+        r = self.client.post(url, dict(rfc='rfc9998', drafts=('draft-ietf-mars-approved-document', )))
+        self.assertEqual(r.status_code, 200)
+        content = unicontent(r)
+        self.assertTrue('Downref is already in the registry' in content)
+
+        # error - source is not in an approved state
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        r = self.client.post(url, dict(rfc='rfc9998', drafts=('draft-ietf-mars-test', )))
+        self.assertEqual(r.status_code, 200)
+        content = unicontent(r)
+        self.assertTrue('Draft is not yet approved' in content)
+
+        # error - the target is not a normative reference of the source
+        draft = Document.objects.get(name="draft-ietf-mars-test")
+        draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="pub"))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        r = self.client.post(url, dict(rfc='rfc9998', drafts=('draft-ietf-mars-test', )))
+        self.assertEqual(r.status_code, 200)
+        content = unicontent(r)
+        self.assertTrue('There does not seem to be a normative reference to RFC' in content)
+        self.assertTrue('Save downref anyway' in content)
+
+        # normal - approve the document so the downref is now okay
+        rfc = DocAlias.objects.get(name="rfc9998")
+        RelatedDocument.objects.create(source=draft, target=rfc, relationship_id='refnorm')
+        draft_de_count_before = draft.docevent_set.count()
+        rfc_de_count_before = rfc.document.docevent_set.count()
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        r = self.client.post(url, dict(rfc='rfc9998', drafts=('draft-ietf-mars-test', )))
+        self.assertEqual(r.status_code, 302)
+        newurl = urlreverse('ietf.doc.views_downref.downref_registry')
+        r = self.client.get(newurl)
+        self.assertEqual(r.status_code, 200)
+        content = unicontent(r)
+        self.assertTrue('<a href="/doc/draft-ietf-mars-test' in content)
+        self.assertTrue(RelatedDocument.objects.filter(source=draft, target=rfc, relationship_id='downref-approval'))
+        self.assertEqual(draft.docevent_set.count(), draft_de_count_before + 1)
+        self.assertEqual(rfc.document.docevent_set.count(), rfc_de_count_before + 1)
+
+    def setUp(self):
+        make_test_data()
+        make_downref_test_data()

ietf/doc/urls.py

@@ -34,7 +34,7 @@
 from django.views.generic import RedirectView
 from django.conf import settings
 
-from ietf.doc import views_search, views_draft, views_ballot, views_status_change, views_doc, views_stats, views_help
+from ietf.doc import views_search, views_draft, views_ballot, views_status_change, views_doc, views_downref, views_stats, views_help
 from ietf.utils.urls import url
 
 session_patterns = [
@@ -54,6 +54,8 @@
     url(r'^start-rfc-status-change/(?:%(name)s/)?$' % settings.URL_REGEXPS, views_status_change.start_rfc_status_change),
     url(r'^iesg/(?P<last_call_only>[A-Za-z0-9.-]+/)?$', views_search.drafts_in_iesg_process),
     url(r'^email-aliases/$', views_doc.email_aliases),
+    url(r'^downref/$', views_downref.downref_registry),
+    url(r'^downref/add/?$', views_downref.downref_registry_add),
     url(r'^stats/newrevisiondocevent/?$', views_stats.chart_newrevisiondocevent),
     url(r'^stats/newrevisiondocevent/conf/?$', views_stats.chart_conf_newrevisiondocevent),
     url(r'^stats/newrevisiondocevent/data/?$', views_stats.chart_data_newrevisiondocevent),