merged forward ^/personal/rjs/explore-extref

 - Legacy-Id: 17840

Author: rjsparks

ietf/doc/management/commands/find_github_backup_info.py

@@ -0,0 +1,30 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+import json
+
+from django.core.management.base import BaseCommand
+
+from ietf.extresource.models import ExtResource
+from ietf.doc.models import DocExtResource
+from ietf.group.models import GroupExtResource
+from ietf.person.models import PersonExtResource
+
+class Command(BaseCommand):
+    help = ('Locate information about gihub repositories to backup')
+
+    def handle(self, *args, **options):
+
+        info_dict = {}
+        for repo in ExtResource.objects.filter(name__slug='github_repo'):
+            if repo not in info_dict:
+                info_dict[repo.value] = []
+
+            for username in DocExtResource.objects.filter(extresource__name__slug='github_username', doc__name__in=repo.docextresource_set.values_list('doc__name',flat=True).distinct()):
+                info_dict[repo.value].push(username.value)
+
+            for username in GroupExtResource.objects.filter(extresource__name__slug='github_username', group__acronym__in=repo.groupextresource_set.values_list('group__acronym',flat=True).distinct()):
+                info_dict[repo.value].push(username.value)
+
+            for username in PersonExtResource.objects.filter(extresource__name__slug='github_username', person_id__in=repo.personextresource_set.values_list('person__id',flat=True).distinct()):
+                info_dict[repo.value].push(username.value)
+
+        print (json.dumps(info_dict))

ietf/doc/migrations/0032_extres.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-04-15 10:20
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import ietf.utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name', '0012_extres'),
+        ('doc', '0031_set_state_for_charters_of_replaced_groups'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DocExtResource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('display_name', models.CharField(blank=True, default='', max_length=255)),
+                ('value', models.CharField(max_length=2083)),
+                ('doc', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='doc.Document')),
+                ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')),
+            ],
+        ),
+    ]

ietf/doc/migrations/0033_populate_docextresources.py

@@ -0,0 +1,116 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-03-19 13:06
+from __future__ import unicode_literals
+
+import re
+
+import debug
+
+from collections import OrderedDict, Counter
+
+from django.db import migrations
+
+from ietf.utils.validators import validate_external_resource_value
+from django.core.exceptions import ValidationError
+
+
+name_map = {
+    "Issue.*":                "tracker",
+    ".*FAQ.*":                "faq",
+    ".*Area Web Page":        "webpage",
+    ".*Wiki":                 "wiki",
+    "Home Page":              "webpage",
+    "Slack.*":                "slack",
+    "Additional .* Web Page": "webpage",
+    "Additional .* Page":     "webpage",
+    "Yang catalog entry.*":   "yc_entry",
+    "Yang impact analysis.*": "yc_impact",
+    "GitHub":                 "github_repo",
+    "Github page":            "github_repo",
+    "GitHub repo.*":          "github_repo",
+    "Github repository.*":    "github_repo",
+    "GitHub notifications":   "github_notify",
+    "GitHub org.*":           "github_org",
+    "GitHub User.*":          "github_username",
+    "GitLab User":            "gitlab_username",
+    "GitLab User Name":       "gitlab_username",
+}
+
+# TODO: Review all the None values below and make sure ignoring the URLs they match is really the right thing to do.
+url_map = OrderedDict({
+   "https?://github\\.com": "github_repo",
+   "https?://trac\\.ietf\\.org/.*/wiki": "wiki",
+   "ietf\\.org.*/trac/wiki": "wiki",
+   "trac.*wiki": "wiki",
+   "www\\.ietf\\.org/mailman" : None,
+   "www\\.ietf\\.org/mail-archive" : None,
+   "mailarchive\\.ietf\\.org" : None,
+   "ietf\\.org/logs": "jabber_log",
+   "ietf\\.org/jabber/logs": "jabber_log",
+   "xmpp:.*?join": "jabber_room",
+   "bell-labs\\.com": None,
+   "html\\.charters": None,
+   "datatracker\\.ietf\\.org": None,
+})
+
+def forward(apps, schema_editor):
+    DocExtResource = apps.get_model('doc', 'DocExtResource')
+    ExtResourceName = apps.get_model('name', 'ExtResourceName')
+    DocumentUrl = apps.get_model('doc', 'DocumentUrl')
+
+    stats = Counter()
+
+    for doc_url in DocumentUrl.objects.all():
+        match_found = False
+        for regext,slug in name_map.items():
+            if re.match(regext, doc_url.desc):
+                match_found = True
+                stats['mapped'] += 1
+                name = ExtResourceName.objects.get(slug=slug)
+                DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url, display_name=doc_url.desc) # TODO: validate this value against name.type
+                break
+        if not match_found:
+            for regext, slug in url_map.items():
+                doc_url.url = doc_url.url.strip()
+                if re.search(regext, doc_url.url):
+                    match_found = True
+                    if slug:
+                        stats['mapped'] +=1
+                        name = ExtResourceName.objects.get(slug=slug)
+                        # Munge the URL if it's the first github repo match
+                        #  Remove "/tree/master" substring if it exists
+                        #  Remove trailing "/issues" substring if it exists
+                        #  Remove "/blob/master/.*" pattern if present
+                        if regext == "https?://github\\.com":
+                            doc_url.url = doc_url.url.replace("/tree/master","")
+                            doc_url.url = re.sub('/issues$', '', doc_url.url)
+                            doc_url.url = re.sub('/blob/master.*$', '', doc_url.url)
+                        try:
+                            validate_external_resource_value(name, doc_url.url)
+                            DocExtResource.objects.create(doc=doc_url.doc, name=name, value=doc_url.url, display_name=doc_url.desc) # TODO: validate this value against name.type
+                        except ValidationError as e: # pyflakes:ignore
+                            debug.show('("Failed validation:", doc_url.url, e)')
+                            stats['failed_validation'] +=1
+                    else:
+                        stats['ignored'] +=1
+                    break
+        if not match_found:
+            debug.show('("Not Mapped:",doc_url.desc, doc_url.tag.slug, doc_url.doc.name, doc_url.url)')
+            stats['not_mapped'] += 1
+    print (stats)
+
+def reverse(apps, schema_editor):
+    DocExtResource = apps.get_model('doc', 'DocExtResource')
+    DocExtResource.objects.all().delete()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('doc', '0032_extres'),
+        ('name', '0013_populate_extres'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse)
+    ]

ietf/doc/models.py

@@ -24,7 +24,7 @@
 from ietf.group.models import Group
 from ietf.name.models import ( DocTypeName, DocTagName, StreamName, IntendedStdLevelName, StdLevelName,
     DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName, ReviewAssignmentStateName, FormalLanguageName,
-    DocUrlTagName)
+    DocUrlTagName, ExtResourceName)
 from ietf.person.models import Email, Person
 from ietf.person.utils import get_active_balloters
 from ietf.utils import log
@@ -105,6 +105,7 @@ class DocumentInfo(models.Model):
     note = models.TextField(blank=True)
     internal_comments = models.TextField(blank=True)
 
+
     def file_extension(self):
         if not hasattr(self, '_cached_extension'):
             if self.uploaded_filename:
@@ -861,6 +862,12 @@ class DocumentURL(models.Model):
     desc = models.CharField(max_length=255, default='', blank=True)
     url  = models.URLField(max_length=2083) # 2083 is the legal max for URLs
 
+class DocExtResource(models.Model):
+    doc = ForeignKey(Document) # Should this really be to DocumentInfo rather than Document?
+    name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE)
+    display_name = models.CharField(max_length=255, default='', blank=True)
+    value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length
+
 class RelatedDocHistory(models.Model):
     source = ForeignKey('DocHistory')
     target = ForeignKey('DocAlias', related_name="reversely_related_document_history_set")

ietf/doc/resources.py

@@ -17,7 +17,7 @@
     InitialReviewDocEvent, DocHistoryAuthor, BallotDocEvent, RelatedDocument,
     RelatedDocHistory, BallotPositionDocEvent, AddedMessageEvent, SubmissionDocEvent,
     ReviewRequestDocEvent, ReviewAssignmentDocEvent, EditedAuthorsDocEvent, DocumentURL,
-    IanaExpertDocEvent, IRSGBallotDocEvent )
+    IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource )
 
 from ietf.name.resources import BallotPositionNameResource, DocTypeNameResource
 class BallotTypeResource(ModelResource):
@@ -767,3 +767,23 @@ class Meta:
             "ballotdocevent_ptr": ALL_WITH_RELATIONS,
         }
 api.doc.register(IRSGBallotDocEventResource())
+
+
+from ietf.name.resources import ExtResourceNameResource
+class DocExtResourceResource(ModelResource):
+    doc              = ToOneField(DocumentResource, 'doc')
+    name             = ToOneField(ExtResourceNameResource, 'name')
+    class Meta:
+        queryset = DocExtResource.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'docextresource'
+        ordering = ['id', ]
+        filtering = { 
+            "id": ALL,
+            "display_name": ALL,
+            "value": ALL,
+            "doc": ALL_WITH_RELATIONS,
+            "name": ALL_WITH_RELATIONS,
+        }
+api.doc.register(DocExtResourceResource())

ietf/doc/tests_draft.py

@@ -1123,6 +1123,45 @@ def test_doc_change_document_urls(self):
         self.assertIn('wiki https://wiki.org/', doc.latest_event(DocEvent,type="changed_document").desc)
         self.assertIn('https://wiki.org/', [ u.url for u in doc.documenturl_set.all() ])
 
+    def test_edit_doc_extresources(self):
+        url = urlreverse('ietf.doc.views_draft.edit_doc_extresources', kwargs=dict(name=self.docname))
+
+        login_testing_unauthorized(self, "secretary", url)
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,200)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q('form textarea[id=id_resources]')),1)
+
+        badlines = (
+            'github_repo https://github3.com/some/repo',
+            'github_notify  badaddr',
+            'website /not/a/good/url'
+            'notavalidtag blahblahblah'
+        )
+
+        for line in badlines:
+            r = self.client.post(url, dict(resources=line, submit="1"))
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            self.assertTrue(q('.alert-danger'))
+
+        goodlines = """
+            github_repo https://github.com/some/repo Some display text
+            github_notify notify@example.com
+            github_username githubuser
+            website http://example.com/http/is/fine
+        """
+
+        r = self.client.post(url, dict(resources=goodlines, submit="1"))
+        self.assertEqual(r.status_code,302)
+        doc = Document.objects.get(name=self.docname)
+        self.assertEqual(doc.latest_event(DocEvent,type="changed_document").desc[:35], 'Changed document external resources')
+        self.assertIn('github_username githubuser', doc.latest_event(DocEvent,type="changed_document").desc)
+        self.assertEqual(doc.docextresource_set.count(), 4)
+        self.assertEqual(doc.docextresource_set.get(name__slug='github_repo').display_name, 'Some display text')
+
+
 class SubmitToIesgTests(TestCase):
 
     def setUp(self):

ietf/doc/urls.py

@@ -129,6 +129,7 @@
     url(r'^%(name)s/edit/approvedownrefs/$' % settings.URL_REGEXPS, views_ballot.approve_downrefs),
     url(r'^%(name)s/edit/makelastcall/$' % settings.URL_REGEXPS, views_ballot.make_last_call),
     url(r'^%(name)s/edit/urls/$' % settings.URL_REGEXPS, views_draft.edit_document_urls),
+    url(r'^%(name)s/edit/resources/$' % settings.URL_REGEXPS, views_draft.edit_doc_extresources),
     url(r'^%(name)s/edit/issueballot/irsg/$' % settings.URL_REGEXPS, views_ballot.issue_irsg_ballot),
     url(r'^%(name)s/edit/closeballot/irsg/$' % settings.URL_REGEXPS, views_ballot.close_irsg_ballot),
 

ietf/doc/views_draft.py

@@ -43,11 +43,12 @@
 from ietf.ietfauth.utils import has_role, is_authorized_in_doc_stream, user_is_person, is_individual_draft_author
 from ietf.ietfauth.utils import role_required
 from ietf.message.models import Message
-from ietf.name.models import IntendedStdLevelName, DocTagName, StreamName, DocUrlTagName
+from ietf.name.models import IntendedStdLevelName, DocTagName, StreamName, DocUrlTagName, ExtResourceName
 from ietf.person.fields import SearchableEmailField
 from ietf.person.models import Person, Email
 from ietf.utils.mail import send_mail, send_mail_message, on_behalf_of
 from ietf.utils.textupload import get_cleaned_text_file_content
+from ietf.utils.validators import validate_external_resource_value
 from ietf.utils import log
 from ietf.mailtrigger.utils import gather_address_lists
 
@@ -1253,6 +1254,88 @@ def format_urls(urls, fs="\n"):
     title = "Additional document URLs"
     return render(request, 'doc/edit_field.html',dict(doc=doc, form=form, title=title, info=info) )
 
+
+def edit_doc_extresources(request, name):
+    class DocExtResourceForm(forms.Form):
+        resources = forms.CharField(widget=forms.Textarea, label="Additional Resources", required=False,
+            help_text=("Format: 'tag value (Optional description)'."
+                " Separate multiple entries with newline. When the value is a URL, use https:// where possible.") )
+
+        def clean_resources(self):
+            lines = [x.strip() for x in self.cleaned_data["resources"].splitlines() if x.strip()]
+            errors = []
+            for l in lines:
+                parts = l.split()
+                if len(parts) == 1:
+                    errors.append("Too few fields: Expected at least tag and value: '%s'" % l)
+                elif len(parts) >= 2:
+                    name_slug = parts[0]
+                    try:
+                        name = ExtResourceName.objects.get(slug=name_slug)
+                    except ObjectDoesNotExist:
+                        errors.append("Bad tag in '%s': Expected one of %s" % (l, ', '.join([ o.slug for o in ExtResourceName.objects.all() ])))
+                        continue
+                    value = parts[1]
+                    try:
+                        validate_external_resource_value(name, value)
+                    except ValidationError as e:
+                        e.message += " : " + value
+                        errors.append(e)
+            if errors:
+                raise ValidationError(errors)
+            return lines
+
+    def format_resources(resources, fs="\n"):
+        res = []
+        for r in resources:
+            if r.display_name:
+                res.append("%s %s (%s)" % (r.name.slug, r.value, r.display_name.strip('()')))
+            else:
+                res.append("%s %s" % (r.name.slug, r.value)) 
+                # TODO: This is likely problematic if value has spaces. How then to delineate value and display_name? Perhaps in the short term move to comma or pipe separation.
+                # Might be better to shift to a formset instead of parsing these lines.
+        return fs.join(res)
+
+    doc = get_object_or_404(Document, name=name)
+
+    if not (has_role(request.user, ("Secretariat", "Area Director"))
+            or is_authorized_in_doc_stream(request.user, doc)
+            or is_individual_draft_author(request.user, doc)):
+        return HttpResponseForbidden("You do not have the necessary permissions to view this page")
+
+    old_resources = format_resources(doc.docextresource_set.all())
+
+    if request.method == 'POST':
+        form = DocExtResourceForm(request.POST)
+        if form.is_valid():
+            old_resources = sorted(old_resources.splitlines())
+            new_resources = sorted(form.cleaned_data['resources'])
+            if old_resources != new_resources:
+                doc.docextresource_set.all().delete()
+                for u in new_resources:
+                    parts = u.split(None, 2)
+                    name = parts[0]
+                    value = parts[1]
+                    display_name = ' '.join(parts[2:]).strip('()')
+                    doc.docextresource_set.create(value=value, name_id=name, display_name=display_name)
+                new_resources = format_resources(doc.docextresource_set.all())
+                e = DocEvent(doc=doc, rev=doc.rev, by=request.user.person, type='changed_document')
+                e.desc = "Changed document external resources from:\n\n%s\n\nto:\n\n%s" % (old_resources, new_resources)
+                e.save()
+                doc.save_with_history([e])
+                messages.success(request,"Document resources updated.")
+            else:
+                messages.info(request,"No change in Document resources.")
+            return redirect('ietf.doc.views_doc.document_main', name=doc.name)
+    else:
+        form = DocExtResourceForm(initial={'resources': old_resources, })
+
+    info = "Valid tags:<br><br> %s" % ', '.join([ o.slug for o in ExtResourceName.objects.all().order_by('slug') ])
+    # May need to explain the tags more - probably more reason to move to a formset.
+    title = "Additional document resources"
+    return render(request, 'doc/edit_field.html',dict(doc=doc, form=form, title=title, info=info) )
+
+
 def request_publication(request, name):
     """Request publication by RFC Editor for a document which hasn't
     been through the IESG ballot process."""