Added display-name. Refined migrations.

 - Legacy-Id: 17646

Author: rjsparks

ietf/doc/migrations/0032_extres.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.29 on 2020-03-22 10:49
+# Generated by Django 1.11.29 on 2020-04-15 10:20
 from __future__ import unicode_literals
 
 from django.db import migrations, models
@@ -19,6 +19,7 @@ class Migration(migrations.Migration):
             name='DocExtResource',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('display_name', models.CharField(blank=True, default='', max_length=255)),
                 ('value', models.CharField(max_length=2083)),
                 ('doc', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='doc.Document')),
                 ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')),

ietf/doc/migrations/0033_populate_docextresources.py

@@ -11,16 +11,6 @@
 
 from django.db import migrations
 
-"""
-This makes me very nervous:
-
->>> DocumentURL.objects.filter(desc__icontains='notifications').values_list('tag',flat=True).distinct()
-<QuerySet ['yang-impact-analysis', 'yang-module-metadata']>
-
-I suspect the wrong thing is happening with the map below wrt the GitHub notificaitons string.
-
-"""
-
 name_map = {
     "Issue.*":                "tracker",
     ".*FAQ.*":                "faq",
@@ -76,10 +66,11 @@ def forward(apps, schema_editor):
                 match_found = True
                 mapped += 1
                 name = ExtResourceName.objects.get(slug=slug)
-                DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url) # TODO: validate this value against name.type
+                DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url, display_name=doc_url.desc) # TODO: validate this value against name.type
                 break
         if not match_found:
             for regext, slug in url_map.items():
+                doc_url.url = doc_url.url.strip()
                 if re.search(regext, doc_url.url):
                     match_found = True
                     if slug:
@@ -93,7 +84,7 @@ def forward(apps, schema_editor):
                             doc_url.url = doc_url.url.replace("/tree/master","")
                             doc_url.url = re.sub('/issues$', '', doc_url.url)
                             doc_url.url = re.sub('/blob/master.*$', '', doc_url.url)
-                        DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url) # TODO: validate this value against name.type
+                        DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url, display_name=doc_url.desc) # TODO: validate this value against name.type
                     else:
                         ignored +=1
                     break

ietf/doc/models.py

@@ -865,6 +865,7 @@ class DocumentURL(models.Model):
 class DocExtResource(models.Model):
     doc = ForeignKey(Document) # Should this really be to DocumentInfo rather than Document?
     name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE)
+    display_name = models.CharField(max_length=255, default='', blank=True)
     value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length
 
 @python_2_unicode_compatible

ietf/group/migrations/0024_extres.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.29 on 2020-03-22 10:49
+# Generated by Django 1.11.29 on 2020-04-15 10:20
 from __future__ import unicode_literals
 
 from django.db import migrations, models
@@ -19,6 +19,7 @@ class Migration(migrations.Migration):
             name='GroupExtResource',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('display_name', models.CharField(blank=True, default='', max_length=255)),
                 ('value', models.CharField(max_length=2083)),
                 ('group', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='group.Group')),
                 ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')),

ietf/group/migrations/0025_populate_groupextresources.py

@@ -11,7 +11,6 @@
 
 from django.db import migrations
 
-
 name_map = {
     "Issue.*":                "tracker",
     ".*FAQ.*":                "faq",
@@ -34,21 +33,29 @@
     "GitLab User Name":       "gitlab_username",
 }
 
-# TODO: Review all the None values below and make sure ignoring the URLs they match is really the right thing to do.
+# TODO: Consider dropping known bad links at this point
+#   " *https?://www.ietf.org/html.charters/*": None, # all these links are dead
+#   " *http://www.bell-labs.com/mailing-lists/pint": None, # dead link
+#   "http://www.ietf.org/wg/videos/mile-overview.html": None, # dead link
+#   " http://domen.uninett.no/~hta/ietf/notary-status.h": None,  # dead link
+#   " http://www.ERC.MsState.Edu/packetway": None, # dead link
+#   "mailarchive\\.ietf\\.org" : None,
+#   "bell-labs\\.com": None,
+#   "html\\.charters": None,
+#   "datatracker\\.ietf\\.org": None,
+#   etc.
+
 url_map = OrderedDict({
    "https?://github\\.com": "github_repo",
    "https?://trac\\.ietf\\.org/.*/wiki": "wiki",
    "ietf\\.org.*/trac/wiki": "wiki",
    "trac.*wiki": "wiki",
-   "www\\.ietf\\.org/mailman" : None,
-   "www\\.ietf\\.org/mail-archive" : None,
-   "mailarchive\\.ietf\\.org" : None,
+   "www\\.ietf\\.org/mailman" : "mailing_list",
+   "www\\.ietf\\.org/mail-archive" : "mailing_list_archive",
    "ietf\\.org/logs": "jabber_log",
    "ietf\\.org/jabber/logs": "jabber_log",
    "xmpp:.*?join": "jabber_room",
-   "bell-labs\\.com": None,
-   "html\\.charters": None,
-   "datatracker\\.ietf\\.org": None,
+   "https?://.*": "webpage"
 })
 
 def forward(apps, schema_editor):
@@ -60,17 +67,19 @@ def forward(apps, schema_editor):
     not_mapped = 0
     ignored = 0
 
+    debug.say("Matching...")
     for group_url in GroupUrl.objects.all():
         match_found = False
         for regext,slug in name_map.items():
             if re.match(regext, group_url.name):
                 match_found = True
                 mapped += 1
                 name = ExtResourceName.objects.get(slug=slug)
-                GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url) # TODO: validate this value against name.type
+                GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url, display_name=group_url.name) # TODO: validate this value against name.type
                 break
         if not match_found:
             for regext, slug in url_map.items():
+                group_url.url = group_url.url.strip()
                 if re.search(regext, group_url.url):
                     match_found = True
                     if slug:
@@ -84,7 +93,7 @@ def forward(apps, schema_editor):
                             group_url.url = group_url.url.replace("/tree/master","")
                             group_url.url = re.sub('/issues$', '', group_url.url)
                             group_url.url = re.sub('/blob/master.*$', '', group_url.url)
-                        GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url) # TODO: validate this value against name.type
+                        GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url, display_name=group_url.name) # TODO: validate this value against name.type
                     else:
                         ignored +=1
                     break

ietf/group/models.py

@@ -266,6 +266,7 @@ def __str__(self):
 class GroupExtResource(models.Model):
     group = ForeignKey(Group) # Should this really be to GroupInfo?
     name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE)
+    display_name = models.CharField(max_length=255, default='', blank=True)
     value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length
 
 @python_2_unicode_compatible

ietf/name/migrations/0011_populate_extres.py

@@ -34,6 +34,8 @@ def forward(apps, schema_editor):
         resourcename("yc_impact","Yang Impact Analysis", "url"),
         resourcename("jabber_room","Jabber Room", "url"),
         resourcename("jabber_log","Jabber Log", "url"),
+        resourcename("mailing_list","Mailing List", "url"),
+        resourcename("mailing_list_archive","Mailing List Archive","url"),
     ]
 
     for name in resourcenames:

ietf/person/migrations/0010_extres.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.29 on 2020-03-22 10:49
+# Generated by Django 1.11.29 on 2020-04-15 10:20
 from __future__ import unicode_literals
 
 from django.db import migrations, models
@@ -19,6 +19,7 @@ class Migration(migrations.Migration):
             name='PersonExtResource',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('display_name', models.CharField(blank=True, default='', max_length=255)),
                 ('value', models.CharField(max_length=2083)),
                 ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')),
                 ('person', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='person.Person')),

ietf/person/models.py

@@ -242,6 +242,7 @@ def json_dict(self, hostscheme):
 class PersonExtResource(models.Model):
     person = ForeignKey(Person) 
     name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE)
+    display_name = models.CharField(max_length=255, default='', blank=True)
     value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length
 
 @python_2_unicode_compatible