Merged in the ExtResource work from rjsparks@nostrum.com, based on a feature request and contributions from rsalz@akamai.com. This provides external resource models for Person, Group, and Document models, as a generalisation of the previous related-URL concept. This provides a consistent labelling and classification of URLs and other resources related to a Group, Document, or Person. The GroupURL (and similar) classes will be removed in a later step. Some DocumentURL instances (such as auth48 URLs) remain to convert, as they have come in with other merged-in work while the ExtResource work was in transit.

 - Legacy-Id: 18192

Author: levkowetz

ietf/doc/admin.py

@@ -11,8 +11,9 @@
     StateDocEvent, ConsensusDocEvent, BallotType, BallotDocEvent, WriteupDocEvent, LastCallDocEvent,
     TelechatDocEvent, BallotPositionDocEvent, ReviewRequestDocEvent, InitialReviewDocEvent,
     AddedMessageEvent, SubmissionDocEvent, DeletedEvent, EditedAuthorsDocEvent, DocumentURL,
-    ReviewAssignmentDocEvent, IanaExpertDocEvent, IRSGBallotDocEvent )
+    ReviewAssignmentDocEvent, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource )
 
+from ietf.utils.validators import validate_external_resource_value
 
 class StateTypeAdmin(admin.ModelAdmin):
     list_display = ["slug", "label"]
@@ -183,3 +184,14 @@ class DocumentUrlAdmin(admin.ModelAdmin):
     search_fields = ['doc__name', 'url', ]
     raw_id_fields = ['doc', ]
 admin.site.register(DocumentURL, DocumentUrlAdmin)
+
+class DocExtResourceAdminForm(forms.ModelForm):
+    def clean(self):
+        validate_external_resource_value(self.cleaned_data['name'],self.cleaned_data['value'])
+
+class DocExtResourceAdmin(admin.ModelAdmin):
+    form = DocExtResourceAdminForm
+    list_display = ['id', 'doc', 'name', 'display_name', 'value',]
+    search_fields = ['doc__name', 'value', 'display_name', 'name__slug',]
+    raw_id_fields = ['doc', ]
+admin.site.register(DocExtResource, DocExtResourceAdmin)

ietf/doc/management/commands/find_github_backup_info.py

@@ -0,0 +1,45 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+
+from django.core.management.base import BaseCommand
+from django.db.models import F
+
+from ietf.doc.models import DocExtResource
+from ietf.group.models import GroupExtResource
+from ietf.person.models import PersonExtResource
+
+class Command(BaseCommand):
+    help = ('Locate information about gihub repositories to backup')
+
+    def handle(self, *args, **options):
+
+        info_dict = {}
+
+
+        for repo in DocExtResource.objects.filter(name__slug='github_repo'):
+            if not repo.value.endswith('/'):
+                repo.value += '/'
+            if repo not in info_dict:
+                info_dict[repo.value] = []
+            for username in DocExtResource.objects.filter(name__slug='github_username', doc=F('doc')):
+                info_dict[repo.value].push(username.value)
+
+        for repo in GroupExtResource.objects.filter(name__slug='github_repo'):
+            if not repo.value.endswith('/'):
+                repo.value += '/'
+            if repo not in info_dict:
+                info_dict[repo.value] = []
+            for username in GroupExtResource.objects.filter(name__slug='github_username', group=F('group')):
+                info_dict[repo.value].push(username.value)
+
+        for repo in PersonExtResource.objects.filter(name__slug='github_repo'):
+            if not repo.value.endswith('/'):
+                repo.value += '/'
+            if repo not in info_dict:
+                info_dict[repo.value] = []
+            for username in PersonExtResource.objects.filter(name__slug='github_username', person=F('person')):
+                info_dict[repo.value].push(username.value)
+
+        #print (json.dumps(info_dict))
+        # For now, all we need are the repo names
+        for name in info_dict.keys():
+            print(name)

ietf/doc/migrations/0034_extres.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-04-15 10:20
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import ietf.utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name', '0014_extres'),
+        ('doc', '0033_populate_auth48_urls'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DocExtResource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('display_name', models.CharField(blank=True, default='', max_length=255)),
+                ('value', models.CharField(max_length=2083)),
+                ('doc', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='doc.Document')),
+                ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')),
+            ],
+        ),
+    ]

ietf/doc/migrations/0035_populate_docextresources.py

@@ -0,0 +1,125 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-03-19 13:06
+from __future__ import unicode_literals
+
+import re
+
+import debug # pyflakes:ignore
+
+from collections import OrderedDict, Counter
+from io import StringIO
+
+from django.db import migrations
+
+from ietf.utils.validators import validate_external_resource_value
+from django.core.exceptions import ValidationError
+
+
+name_map = {
+    "Issue.*":                "tracker",
+    ".*FAQ.*":                "faq",
+    ".*Area Web Page":        "webpage",
+    ".*Wiki":                 "wiki",
+    "Home Page":              "webpage",
+    "Slack.*":                "slack",
+    "Additional .* Web Page": "webpage",
+    "Additional .* Page":     "webpage",
+    "Yang catalog entry.*":   "yc_entry",
+    "Yang impact analysis.*": "yc_impact",
+    "GitHub":                 "github_repo",
+    "Github page":            "github_repo",
+    "GitHub repo.*":          "github_repo",
+    "Github repository.*":    "github_repo",
+    "GitHub org.*":           "github_org",
+    "GitHub User.*":          "github_username",
+    "GitLab User":            "gitlab_username",
+    "GitLab User Name":       "gitlab_username",
+}
+
+url_map = OrderedDict({
+   "https?://github\\.com": "github_repo",
+   "https://git.sr.ht/": "repo",
+   "https://todo.sr.ht/": "tracker",
+   "https?://trac\\.ietf\\.org/.*/wiki": "wiki",
+   "ietf\\.org.*/trac/wiki": "wiki",
+   "trac.*wiki": "wiki",
+   "www\\.ietf\\.org/mailman" : None,
+   "www\\.ietf\\.org/mail-archive" : None,
+   "mailarchive\\.ietf\\.org" : None,
+   "ietf\\.org/logs": "jabber_log",
+   "ietf\\.org/jabber/logs": "jabber_log",
+   "xmpp:.*?join": "jabber_room",
+   "bell-labs\\.com": None,
+   "html\\.charters": None,
+   "datatracker\\.ietf\\.org": None,
+})
+
+def forward(apps, schema_editor):
+    DocExtResource = apps.get_model('doc', 'DocExtResource')
+    ExtResourceName = apps.get_model('name', 'ExtResourceName')
+    DocumentUrl = apps.get_model('doc', 'DocumentUrl')
+
+    stats = Counter()
+    stats_file = StringIO()
+
+    for doc_url in DocumentUrl.objects.all():
+        doc_url.url = doc_url.url.strip()
+        match_found = False
+        for regext,slug in name_map.items():
+            if re.fullmatch(regext, doc_url.desc):
+                match_found = True
+                stats['mapped'] += 1
+                name = ExtResourceName.objects.get(slug=slug)
+                try:
+                    validate_external_resource_value(name, doc_url.url)
+                    DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url, display_name=doc_url.desc)
+                except ValidationError as e: # pyflakes:ignore
+                    print("Failed validation:", doc_url.url, e, file=stats_file)
+                    stats['failed_validation'] +=1
+                break
+        if not match_found:
+            for regext, slug in url_map.items():
+                if re.search(regext, doc_url.url):
+                    match_found = True
+                    if slug:
+                        stats['mapped'] +=1
+                        name = ExtResourceName.objects.get(slug=slug)
+                        # Munge the URL if it's the first github repo match
+                        #  Remove "/tree/master" substring if it exists
+                        #  Remove trailing "/issues" substring if it exists
+                        #  Remove "/blob/master/.*" pattern if present
+                        if regext == "https?://github\\.com":
+                            doc_url.url = doc_url.url.replace("/tree/master","")
+                            doc_url.url = re.sub('/issues$', '', doc_url.url)
+                            doc_url.url = re.sub('/blob/master.*$', '', doc_url.url)
+                        try:
+                            validate_external_resource_value(name, doc_url.url)
+                            DocExtResource.objects.create(doc=doc_url.doc, name=name, value=doc_url.url, display_name=doc_url.desc) 
+                        except ValidationError as e: # pyflakes:ignore
+                            print("Failed validation:", doc_url.url, e, file=stats_file)
+                            stats['failed_validation'] +=1
+                    else:
+                        stats['ignored'] +=1
+                    break
+        if not match_found:
+            print("Not Mapped:", doc_url.desc, doc_url.tag.slug, doc_url.doc.name, doc_url.url, file=stats_file)
+            stats['not_mapped'] += 1
+    print('')
+    print(stats_file.getvalue())
+    print (stats)
+
+def reverse(apps, schema_editor):
+    DocExtResource = apps.get_model('doc', 'DocExtResource')
+    DocExtResource.objects.all().delete()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('doc', '0034_extres'),
+        ('name', '0015_populate_extres'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse)
+    ]

ietf/doc/models.py

@@ -24,7 +24,7 @@
 from ietf.group.models import Group
 from ietf.name.models import ( DocTypeName, DocTagName, StreamName, IntendedStdLevelName, StdLevelName,
     DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName, ReviewAssignmentStateName, FormalLanguageName,
-    DocUrlTagName)
+    DocUrlTagName, ExtResourceName)
 from ietf.person.models import Email, Person
 from ietf.person.utils import get_active_balloters
 from ietf.utils import log
@@ -862,6 +862,15 @@ class DocumentURL(models.Model):
     desc = models.CharField(max_length=255, default='', blank=True)
     url  = models.URLField(max_length=2083) # 2083 is the legal max for URLs
 
+class DocExtResource(models.Model):
+    doc = ForeignKey(Document) # Should this really be to DocumentInfo rather than Document?
+    name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE)
+    display_name = models.CharField(max_length=255, default='', blank=True)
+    value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length
+    def __str__(self):
+        priority = self.display_name or self.name.name
+        return u"%s (%s) %s" % (priority, self.name.slug, self.value)
+
 class RelatedDocHistory(models.Model):
     source = ForeignKey('DocHistory')
     target = ForeignKey('DocAlias', related_name="reversely_related_document_history_set")

ietf/doc/resources.py

@@ -17,7 +17,7 @@
     InitialReviewDocEvent, DocHistoryAuthor, BallotDocEvent, RelatedDocument,
     RelatedDocHistory, BallotPositionDocEvent, AddedMessageEvent, SubmissionDocEvent,
     ReviewRequestDocEvent, ReviewAssignmentDocEvent, EditedAuthorsDocEvent, DocumentURL,
-    IanaExpertDocEvent, IRSGBallotDocEvent )
+    IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource )
 
 from ietf.name.resources import BallotPositionNameResource, DocTypeNameResource
 class BallotTypeResource(ModelResource):
@@ -767,3 +767,23 @@ class Meta:
             "ballotdocevent_ptr": ALL_WITH_RELATIONS,
         }
 api.doc.register(IRSGBallotDocEventResource())
+
+
+from ietf.name.resources import ExtResourceNameResource
+class DocExtResourceResource(ModelResource):
+    doc              = ToOneField(DocumentResource, 'doc')
+    name             = ToOneField(ExtResourceNameResource, 'name')
+    class Meta:
+        queryset = DocExtResource.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        resource_name = 'docextresource'
+        ordering = ['id', ]
+        filtering = { 
+            "id": ALL,
+            "display_name": ALL,
+            "value": ALL,
+            "doc": ALL_WITH_RELATIONS,
+            "name": ALL_WITH_RELATIONS,
+        }
+api.doc.register(DocExtResourceResource())

ietf/doc/tests_draft.py

@@ -1104,24 +1104,44 @@ def test_doc_change_shepherd_writeup(self):
         q = PyQuery(r.content)
         self.assertTrue(q('textarea')[0].text.strip().startswith("As required by RFC 4858"))
 
-    def test_doc_change_document_urls(self):
-        url = urlreverse('ietf.doc.views_draft.edit_document_urls', kwargs=dict(name=self.docname))
-  
-        # get
+    def test_edit_doc_extresources(self):
+        url = urlreverse('ietf.doc.views_draft.edit_doc_extresources', kwargs=dict(name=self.docname))
+
         login_testing_unauthorized(self, "secretary", url)
 
         r = self.client.get(url)
         self.assertEqual(r.status_code,200)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('form textarea[id=id_urls]')),1)
+        self.assertEqual(len(q('form textarea[id=id_resources]')),1)
 
-        # direct edit
-        r = self.client.post(url, dict(urls='wiki https://wiki.org/ Wiki\nrepository https://repository.org/ Repo\n', submit="1"))
+        badlines = (
+            'github_repo https://github3.com/some/repo',
+            'github_notify  badaddr',
+            'website /not/a/good/url',
+            'notavalidtag blahblahblah',
+        )
+
+        for line in badlines:
+            r = self.client.post(url, dict(resources=line, submit="1"))
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            self.assertTrue(q('.alert-danger'))
+
+        goodlines = """
+            github_repo https://github.com/some/repo Some display text
+            github_username githubuser
+            webpage http://example.com/http/is/fine
+        """
+
+        r = self.client.post(url, dict(resources=goodlines, submit="1"))
         self.assertEqual(r.status_code,302)
         doc = Document.objects.get(name=self.docname)
-        self.assertTrue(doc.latest_event(DocEvent,type="changed_document").desc.startswith('Changed document URLs'))
-        self.assertIn('wiki https://wiki.org/', doc.latest_event(DocEvent,type="changed_document").desc)
-        self.assertIn('https://wiki.org/', [ u.url for u in doc.documenturl_set.all() ])
+        self.assertEqual(doc.latest_event(DocEvent,type="changed_document").desc[:35], 'Changed document external resources')
+        self.assertIn('github_username githubuser', doc.latest_event(DocEvent,type="changed_document").desc)
+        self.assertEqual(doc.docextresource_set.count(), 3)
+        self.assertEqual(doc.docextresource_set.get(name__slug='github_repo').display_name, 'Some display text')
+        self.assertIn(doc.docextresource_set.first().name.slug,str(doc.docextresource_set.first()))
+
 
 class SubmitToIesgTests(TestCase):
 

ietf/doc/urls.py

@@ -128,7 +128,7 @@
     url(r'^%(name)s/edit/approveballot/$' % settings.URL_REGEXPS, views_ballot.approve_ballot),
     url(r'^%(name)s/edit/approvedownrefs/$' % settings.URL_REGEXPS, views_ballot.approve_downrefs),
     url(r'^%(name)s/edit/makelastcall/$' % settings.URL_REGEXPS, views_ballot.make_last_call),
-    url(r'^%(name)s/edit/urls/$' % settings.URL_REGEXPS, views_draft.edit_document_urls),
+    url(r'^%(name)s/edit/resources/$' % settings.URL_REGEXPS, views_draft.edit_doc_extresources),
     url(r'^%(name)s/edit/issueballot/irsg/$' % settings.URL_REGEXPS, views_ballot.issue_irsg_ballot),
     url(r'^%(name)s/edit/closeballot/irsg/$' % settings.URL_REGEXPS, views_ballot.close_irsg_ballot),