Removed extref app. Refined population migrations.

 - Legacy-Id: 17509

Author: rjsparks

…ment/commands/find_github_backup_info.py …ment/commands/find_github_backup_info.pyietf/extresource/management/commands/find_github_backup_info.py renamed to ietf/doc/management/commands/find_github_backup_info.py ietf/extresource/management/commands/find_github_backup_info.py renamed to ietf/doc/management/commands/find_github_backup_info.py

@@ -2,7 +2,6 @@
 import json
 
 from django.core.management.base import BaseCommand
-from django.db.models import Q
 
 from ietf.extresource.models import ExtResource
 from ietf.doc.models import DocExtResource

ietf/doc/migrations/0032_extres.py

@@ -1,6 +1,5 @@
-# Copyright The IETF Trust 2020, All Rights Reserved
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.29 on 2020-03-19 13:56
+# Generated by Django 1.11.29 on 2020-03-22 10:49
 from __future__ import unicode_literals
 
 from django.db import migrations, models
@@ -11,7 +10,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('extresource', '0001_extres'),
+        ('name', '0011_populate_extres'),
         ('doc', '0031_set_state_for_charters_of_replaced_groups'),
     ]
 
@@ -20,8 +19,9 @@ class Migration(migrations.Migration):
             name='DocExtResource',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('value', models.CharField(max_length=2083)),
                 ('doc', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='doc.Document')),
-                ('extresource', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='extresource.ExtResource')),
+                ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')),
             ],
         ),
     ]

ietf/doc/migrations/0033_populate_docextresources.py

@@ -7,6 +7,8 @@
 
 import debug
 
+from collections import OrderedDict
+
 from django.db import migrations
 
 """
@@ -41,14 +43,31 @@
     "GitLab User Name":       "gitlab_username",
 }
 
+# TODO: Review all the None values below and make sure ignoring the URLs they match is really the right thing to do.
+url_map = OrderedDict({
+   "https?://github\\.com": "github_repo",
+   "https?://trac\\.ietf\\.org/.*/wiki": "wiki",
+   "ietf\\.org.*/trac/wiki": "wiki",
+   "trac.*wiki": "wiki",
+   "www\\.ietf\\.org/mailman" : None,
+   "www\\.ietf\\.org/mail-archive" : None,
+   "mailarchive\\.ietf\\.org" : None,
+   "ietf\\.org/logs": "jabber_log",
+   "ietf\\.org/jabber/logs": "jabber_log",
+   "xmpp:.*?join": "jabber_room",
+   "bell-labs\\.com": None,
+   "html\\.charters": None,
+   "datatracker\\.ietf\\.org": None,
+})
+
 def forward(apps, schema_editor):
     DocExtResource = apps.get_model('doc', 'DocExtResource')
-    ExtResource = apps.get_model('extresource', 'ExtResource')
     ExtResourceName = apps.get_model('name', 'ExtResourceName')
     DocumentUrl = apps.get_model('doc', 'DocumentUrl')
 
     mapped = 0
     not_mapped = 0
+    ignored = 0
 
     for doc_url in DocumentUrl.objects.all():
         match_found = False
@@ -57,13 +76,31 @@ def forward(apps, schema_editor):
                 match_found = True
                 mapped += 1
                 name = ExtResourceName.objects.get(slug=slug)
-                ext_res = ExtResource.objects.create(name_id=slug, value= doc_url.url) # TODO: validate this value against name.type
-                DocExtResource.objects.create(doc=doc_url.doc, extresource=ext_res)
+                DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url) # TODO: validate this value against name.type
                 break
+        if not match_found:
+            for regext, slug in url_map.items():
+                if re.search(regext, doc_url.url):
+                    match_found = True
+                    if slug:
+                        mapped +=1
+                        name = ExtResourceName.objects.get(slug=slug)
+                        # Munge the URL if it's the first github repo match
+                        #  Remove "/tree/master" substring if it exists
+                        #  Remove trailing "/issues" substring if it exists
+                        #  Remove "/blob/master/.*" pattern if present
+                        if regext == "https?://github\\.com":
+                            doc_url.url = doc_url.url.replace("/tree/master","")
+                            doc_url.url = re.sub('/issues$', '', doc_url.url)
+                            doc_url.url = re.sub('/blob/master.*$', '', doc_url.url)
+                        DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url) # TODO: validate this value against name.type
+                    else:
+                        ignored +=1
+                    break
         if not match_found:
             debug.show('("Not Mapped:",doc_url.desc, doc_url.tag.slug, doc_url.doc.name, doc_url.url)')
             not_mapped += 1
-    debug.show('(mapped, not_mapped)')
+    debug.show('(mapped, ignored, not_mapped)')
 
 def reverse(apps, schema_editor):
     DocExtResource = apps.get_model('doc', 'DocExtResource')
@@ -73,7 +110,6 @@ class Migration(migrations.Migration):
 
     dependencies = [
         ('doc', '0032_extres'),
-        ('extresource', '0001_extres'),
         ('name', '0011_populate_extres'),
     ]
 

ietf/doc/models.py

@@ -22,11 +22,10 @@
 
 import debug                            # pyflakes:ignore
 
-from ietf.extresource.models import ExtResource
 from ietf.group.models import Group
 from ietf.name.models import ( DocTypeName, DocTagName, StreamName, IntendedStdLevelName, StdLevelName,
     DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName, ReviewAssignmentStateName, FormalLanguageName,
-    DocUrlTagName)
+    DocUrlTagName, ExtResourceName)
 from ietf.person.models import Email, Person
 from ietf.person.utils import get_active_balloters
 from ietf.utils import log
@@ -865,7 +864,8 @@ class DocumentURL(models.Model):
 
 class DocExtResource(models.Model):
     doc = ForeignKey(Document) # Should this really be to DocumentInfo rather than Document?
-    extresource = ForeignKey(ExtResource)
+    name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE)
+    value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length
 
 @python_2_unicode_compatible
 class RelatedDocHistory(models.Model):