Cleaned up created names. Improved validation during migration. Cleaned up migration output. Cleaned the last of the awp includes from secr views. Removed now unused secr templates. Build extresource urls correctly during submission and when creating group wikis.

 - Legacy-Id: 18158

Author: rjsparks

ietf/doc/migrations/0034_populate_docextresources.py

@@ -5,9 +5,10 @@
 
 import re
 
-import debug
+import debug # pyflakes:ignore
 
 from collections import OrderedDict, Counter
+from io import StringIO
 
 from django.db import migrations
 
@@ -30,14 +31,12 @@
     "Github page":            "github_repo",
     "GitHub repo.*":          "github_repo",
     "Github repository.*":    "github_repo",
-    "GitHub notifications":   "github_notify",
     "GitHub org.*":           "github_org",
     "GitHub User.*":          "github_username",
     "GitLab User":            "gitlab_username",
     "GitLab User Name":       "gitlab_username",
 }
 
-# TODO: Review all the None values below and make sure ignoring the URLs they match is really the right thing to do.
 url_map = OrderedDict({
    "https?://github\\.com": "github_repo",
    "https://git.sr.ht/": "repo",
@@ -62,19 +61,25 @@ def forward(apps, schema_editor):
     DocumentUrl = apps.get_model('doc', 'DocumentUrl')
 
     stats = Counter()
+    stats_file = StringIO()
 
     for doc_url in DocumentUrl.objects.all():
+        doc_url.url = doc_url.url.strip()
         match_found = False
         for regext,slug in name_map.items():
-            if re.match(regext, doc_url.desc):
+            if re.fullmatch(regext, doc_url.desc):
                 match_found = True
                 stats['mapped'] += 1
                 name = ExtResourceName.objects.get(slug=slug)
-                DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url, display_name=doc_url.desc) # TODO: validate this value against name.type
+                try:
+                    validate_external_resource_value(name, doc_url.url)
+                    DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url, display_name=doc_url.desc)
+                except ValidationError as e: # pyflakes:ignore
+                    print("Failed validation:", doc_url.url, e, file=stats_file)
+                    stats['failed_validation'] +=1
                 break
         if not match_found:
             for regext, slug in url_map.items():
-                doc_url.url = doc_url.url.strip()
                 if re.search(regext, doc_url.url):
                     match_found = True
                     if slug:
@@ -90,16 +95,18 @@ def forward(apps, schema_editor):
                             doc_url.url = re.sub('/blob/master.*$', '', doc_url.url)
                         try:
                             validate_external_resource_value(name, doc_url.url)
-                            DocExtResource.objects.create(doc=doc_url.doc, name=name, value=doc_url.url, display_name=doc_url.desc) # TODO: validate this value against name.type
+                            DocExtResource.objects.create(doc=doc_url.doc, name=name, value=doc_url.url, display_name=doc_url.desc) 
                         except ValidationError as e: # pyflakes:ignore
-                            debug.show('("Failed validation:", doc_url.url, e)')
+                            print("Failed validation:", doc_url.url, e, file=stats_file)
                             stats['failed_validation'] +=1
                     else:
                         stats['ignored'] +=1
                     break
         if not match_found:
-            debug.show('("Not Mapped:",doc_url.desc, doc_url.tag.slug, doc_url.doc.name, doc_url.url)')
+            print("Not Mapped:",doc_url.desc, doc_url.tag.slug, doc_url.doc.name, doc_url.url, file=stats_file)
             stats['not_mapped'] += 1
+    print('')
+    print(stats_file.getvalue())
     print (stats)
 
 def reverse(apps, schema_editor):

ietf/doc/tests.py

@@ -1056,9 +1056,9 @@ def test_document_bibtex(self):
         self.assertNotIn('day', entry)
 
         april1 = IndividualRfcFactory.create(
-                  stream_id =       'rse',
+                  stream_id =       'ise',
                   states =          [('draft','rfc'),('draft-iesg','pub')],
-                  std_level_id =    'ind',
+                  std_level_id =    'inf',
                   time =            datetime.datetime(1990,0o4,0o1),
               )
         num = april1.rfc_number()

ietf/doc/tests_draft.py

@@ -1129,17 +1129,16 @@ def test_edit_doc_extresources(self):
 
         goodlines = """
             github_repo https://github.com/some/repo Some display text
-            github_notify notify@example.com
             github_username githubuser
-            website http://example.com/http/is/fine
+            webpage http://example.com/http/is/fine
         """
 
         r = self.client.post(url, dict(resources=goodlines, submit="1"))
         self.assertEqual(r.status_code,302)
         doc = Document.objects.get(name=self.docname)
         self.assertEqual(doc.latest_event(DocEvent,type="changed_document").desc[:35], 'Changed document external resources')
         self.assertIn('github_username githubuser', doc.latest_event(DocEvent,type="changed_document").desc)
-        self.assertEqual(doc.docextresource_set.count(), 4)
+        self.assertEqual(doc.docextresource_set.count(), 3)
         self.assertEqual(doc.docextresource_set.get(name__slug='github_repo').display_name, 'Some display text')
 
 

ietf/group/migrations/0034_populate_groupextresources.py

@@ -5,9 +5,10 @@
 
 import re
 
-import debug
+import debug # pyflakes:ignore
 
 from collections import OrderedDict, Counter
+from io import StringIO
 
 from django.db import migrations
 from django.core.exceptions import ValidationError
@@ -30,25 +31,12 @@
     "Github page":            "github_repo",
     "GitHub repo.*":          "github_repo",
     "Github repository.*":    "github_repo",
-    "GitHub notifications":   "github_notify",
     "GitHub org.*":           "github_org",
     "GitHub User.*":          "github_username",
     "GitLab User":            "gitlab_username",
     "GitLab User Name":       "gitlab_username",
 }
 
-# TODO: Consider dropping known bad links at this point
-#   " *https?://www.ietf.org/html.charters/*": None, # all these links are dead
-#   " *http://www.bell-labs.com/mailing-lists/pint": None, # dead link
-#   "http://www.ietf.org/wg/videos/mile-overview.html": None, # dead link
-#   " http://domen.uninett.no/~hta/ietf/notary-status.h": None,  # dead link
-#   " http://www.ERC.MsState.Edu/packetway": None, # dead link
-#   "mailarchive\\.ietf\\.org" : None,
-#   "bell-labs\\.com": None,
-#   "html\\.charters": None,
-#   "datatracker\\.ietf\\.org": None,
-#   etc.
-
 url_map = OrderedDict({
    "https?://github\\.com": "github_repo",
    "https?://trac\\.ietf\\.org/.*/wiki": "wiki",
@@ -68,19 +56,25 @@ def forward(apps, schema_editor):
     GroupUrl = apps.get_model('group', 'GroupUrl')
 
     stats = Counter()
+    stats_file = StringIO()
 
     for group_url in GroupUrl.objects.all():
+        group_url.url = group_url.url.strip()
         match_found = False
         for regext,slug in name_map.items():
-            if re.match(regext, group_url.name):
+            if re.fullmatch(regext, group_url.name):
                 match_found = True
                 stats['mapped'] += 1
                 name = ExtResourceName.objects.get(slug=slug)
-                GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url, display_name=group_url.name) # TODO: validate this value against name.type
+                try:
+                    validate_external_resource_value(name, group_url.url)
+                    GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url, display_name=group_url.name) # TODO: validate this value against name.type
+                except ValidationError as e: # pyflakes:ignore
+                    print("Failed validation:", group_url.url, e, file=stats_file)
+                    stats['failed_validation'] +=1
                 break
         if not match_found:
             for regext, slug in url_map.items():
-                group_url.url = group_url.url.strip()
                 if re.search(regext, group_url.url):
                     match_found = True
                     if slug:
@@ -96,16 +90,18 @@ def forward(apps, schema_editor):
                             group_url.url = re.sub('/blob/master.*$', '', group_url.url)
                         try:
                             validate_external_resource_value(name, group_url.url)
-                            GroupExtResource.objects.create(group=group_url.group, name=name, value=group_url.url, display_name=group_url.name) # TODO: validate this value against name.type
+                            GroupExtResource.objects.create(group=group_url.group, name=name, value=group_url.url, display_name=group_url.name)
                         except ValidationError as e: # pyflakes:ignore
-                            debug.show('("Failed validation:", group_url.url, e)')
+                            print("Failed validation:", group_url.url, e, file=stats_file)
                             stats['failed_validation'] +=1
                     else:
                         stats['ignored'] +=1
                     break
         if not match_found:
-            debug.show('("Not Mapped:",group_url.group.acronym, group_url.name, group_url.url)')
+            print("Not Mapped:",group_url.group.acronym, group_url.name, group_url.url, file=stats_file)
             stats['not_mapped'] += 1
+    print('')
+    print(stats_file.getvalue())
     print(stats)
 
 def reverse(apps, schema_editor):

ietf/group/tests.py

@@ -74,7 +74,7 @@ def setUp(self):
         set_coverage_checking(False)
         a = WgDraftFactory()
         b = WgDraftFactory()
-        RelatedDocument.objects.create(source=a,target=b.docalias.first(),relationship_id='normref')
+        RelatedDocument.objects.create(source=a,target=b.docalias.first(),relationship_id='refnorm')
 
     def tearDown(self):
         set_coverage_checking(True)

ietf/group/tests_info.py

@@ -658,17 +658,16 @@ def test_edit_extresources(self):
 
         goodlines = """
             github_repo https://github.com/some/repo Some display text
-            github_notify notify@example.com
             github_username githubuser
-            website http://example.com/http/is/fine
+            webpage http://example.com/http/is/fine
         """
 
         r = self.client.post(url, dict(resources=goodlines, submit="1"))
         self.assertEqual(r.status_code,302)
         group = Group.objects.get(acronym=group.acronym)
         self.assertEqual(group.latest_event(GroupEvent,type="info_changed").desc[:20], 'Resources changed to')
         self.assertIn('github_username githubuser', group.latest_event(GroupEvent,type="info_changed").desc)
-        self.assertEqual(group.groupextresource_set.count(), 4)
+        self.assertEqual(group.groupextresource_set.count(), 3)
         self.assertEqual(group.groupextresource_set.get(name__slug='github_repo').display_name, 'Some display text')
 
 

ietf/ietfauth/tests.py

@@ -693,14 +693,13 @@ def test_edit_person_extresources(self):
 
         goodlines = """
             github_repo https://github.com/some/repo Some display text
-            github_notify notify@example.com
             github_username githubuser
-            website http://example.com/http/is/fine
+            webpage http://example.com/http/is/fine
         """
 
         r = self.client.post(url, dict(resources=goodlines, submit="1"))
         self.assertEqual(r.status_code,302)
-        self.assertEqual(person.personextresource_set.count(), 4)
+        self.assertEqual(person.personextresource_set.count(), 3)
         self.assertEqual(person.personextresource_set.get(name__slug='github_repo').display_name, 'Some display text')