datatracker/ietf/group/migrations/0034_populate_groupextresources.py at main · https-github-com-bit/datatracker · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
# Copyright The IETF Trust 2020, All Rights Reserved
# -*- coding: utf-8 -*-
# Generated by Django 1.11.29 on 2020-03-19 13:06
from __future__ import unicode_literals

import re

import debug # pyflakes:ignore

from collections import OrderedDict, Counter
from io import StringIO

from django.db import migrations
from django.core.exceptions import ValidationError


from ietf.utils.validators import validate_external_resource_value

name_map = {
    "Issue.*":                "tracker",
    ".*FAQ.*":                "faq",
    ".*Area Web Page":        "webpage",
    ".*Wiki":                 "wiki",
    "Home Page":              "webpage",
    "Slack.*":                "slack",
    "Additional .* Web Page": "webpage",
    "Additional .* Page":     "webpage",
    "Yang catalog entry.*":   "yc_entry",
    "Yang impact analysis.*": "yc_impact",
    "GitHub":                 "github_repo",
    "Github page":            "github_repo",
    "GitHub repo.*":          "github_repo",
    "Github repository.*":    "github_repo",
    "GitHub org.*":           "github_org",
    "GitHub User.*":          "github_username",
    "GitLab User":            "gitlab_username",
    "GitLab User Name":       "gitlab_username",
}

url_map = OrderedDict({
   "https?://github\\.com": "github_repo",
   "https?://trac\\.ietf\\.org/.*/wiki": "wiki",
   "ietf\\.org.*/trac/wiki": "wiki",
   "trac.*wiki": "wiki",
   "www\\.ietf\\.org/mailman" : "mailing_list",
   "www\\.ietf\\.org/mail-archive" : "mailing_list_archive",
   "ietf\\.org/logs": "jabber_log",
   "ietf\\.org/jabber/logs": "jabber_log",
   "xmpp:.*?join": "jabber_room",
   "https?://.*": "webpage"
})

def forward(apps, schema_editor):
    GroupExtResource = apps.get_model('group', 'GroupExtResource')
    ExtResourceName = apps.get_model('name', 'ExtResourceName')
    GroupUrl = apps.get_model('group', 'GroupUrl')

    stats = Counter()
    stats_file = StringIO()

    for group_url in GroupUrl.objects.all():
        group_url.url = group_url.url.strip()
        match_found = False
        for regext,slug in name_map.items():
            if re.fullmatch(regext, group_url.name):
                match_found = True
                stats['mapped'] += 1
                name = ExtResourceName.objects.get(slug=slug)
                try:
                    validate_external_resource_value(name, group_url.url)
                    GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url, display_name=group_url.name)
                except ValidationError as e: # pyflakes:ignore
                    print("Failed validation:", group_url.url, e, file=stats_file)
                    stats['failed_validation'] +=1
                break
        if not match_found:
            for regext, slug in url_map.items():
                if re.search(regext, group_url.url):
                    match_found = True
                    if slug:
                        stats['mapped'] +=1
                        name = ExtResourceName.objects.get(slug=slug)
                        # Munge the URL if it's the first github repo match
                        #  Remove "/tree/master" substring if it exists
                        #  Remove trailing "/issues" substring if it exists
                        #  Remove "/blob/master/.*" pattern if present
                        if regext == "https?://github\\.com":
                            group_url.url = group_url.url.replace("/tree/master","")
                            group_url.url = re.sub('/issues$', '', group_url.url)
                            group_url.url = re.sub('/blob/master.*$', '', group_url.url)
                        try:
                            validate_external_resource_value(name, group_url.url)
                            GroupExtResource.objects.create(group=group_url.group, name=name, value=group_url.url, display_name=group_url.name)
                        except ValidationError as e: # pyflakes:ignore
                            print("Failed validation:", group_url.url, e, file=stats_file)
                            stats['failed_validation'] +=1
                    else:
                        stats['ignored'] +=1
                    break
        if not match_found:
            print("Not Mapped:",group_url.group.acronym, group_url.name, group_url.url, file=stats_file)
            stats['not_mapped'] += 1
    print('')
    print(stats_file.getvalue())
    print(stats)

def reverse(apps, schema_editor):
    GroupExtResource = apps.get_model('group', 'GroupExtResource')
    GroupExtResource.objects.all().delete()

class Migration(migrations.Migration):

    dependencies = [
        ('group', '0033_extres'),
        ('name', '0015_populate_extres'),
    ]

    operations = [
        migrations.RunPython(forward, reverse)
    ]