Add a suggestion to fix the country name when submitting a draft in

case we can't parse it and it's not empty, add page with list of
countries (to be able to refer people to it), add ISO codes as country
aliases - the country alias code is now more intelligent with respect
to case so it's easier to keep these aliases explicitly
 - Legacy-Id: 12862

Author: OleLaursen

ietf/stats/migrations/0002_add_initial_aliases.py

@@ -18,6 +18,26 @@ def add_affiliation_info(apps, schema_editor):
     AffiliationIgnoredEnding.objects.get_or_create(ending="GmbH\.?")
 
     CountryAlias = apps.get_model("stats", "CountryAlias")
+    for iso_country_code in ['AD', 'AE', 'AF', 'AG', 'AI', 'AL', 'AM', 'AO', 'AQ', 'AR', 'AS', 'AT', 'AU', 'AW',
+                             'AX', 'AZ', 'BA', 'BB', 'BD', 'BE', 'BF', 'BG', 'BH', 'BI', 'BJ', 'BL', 'BM', 'BN',
+                             'BO', 'BQ', 'BR', 'BS', 'BT', 'BV', 'BW', 'BY', 'BZ', 'CA', 'CC', 'CD', 'CF', 'CG',
+                             'CH', 'CI', 'CK', 'CL', 'CM', 'CN', 'CO', 'CR', 'CU', 'CV', 'CW', 'CX', 'CY', 'CZ',
+                             'DE', 'DJ', 'DK', 'DM', 'DO', 'DZ', 'EC', 'EE', 'EG', 'EH', 'ER', 'ES', 'ET', 'FI',
+                             'FJ', 'FK', 'FM', 'FO', 'FR', 'GA', 'GB', 'GD', 'GE', 'GF', 'GG', 'GH', 'GI', 'GL',
+                             'GM', 'GN', 'GP', 'GQ', 'GR', 'GS', 'GT', 'GU', 'GW', 'GY', 'HK', 'HM', 'HN', 'HR',
+                             'HT', 'HU', 'ID', 'IE', 'IL', 'IM', 'IN', 'IO', 'IQ', 'IR', 'IS', 'IT', 'JE', 'JM',
+                             'JO', 'JP', 'KE', 'KG', 'KH', 'KI', 'KM', 'KN', 'KP', 'KR', 'KW', 'KY', 'KZ', 'LA',
+                             'LB', 'LC', 'LI', 'LK', 'LR', 'LS', 'LT', 'LU', 'LV', 'LY', 'MA', 'MC', 'MD', 'ME',
+                             'MF', 'MG', 'MH', 'MK', 'ML', 'MM', 'MN', 'MO', 'MP', 'MQ', 'MR', 'MS', 'MT', 'MU',
+                             'MV', 'MW', 'MX', 'MY', 'MZ', 'NA', 'NC', 'NE', 'NF', 'NG', 'NI', 'NL', 'NO', 'NP',
+                             'NR', 'NU', 'NZ', 'OM', 'PA', 'PE', 'PF', 'PG', 'PH', 'PK', 'PL', 'PM', 'PN', 'PR',
+                             'PS', 'PT', 'PW', 'PY', 'QA', 'RE', 'RO', 'RS', 'RU', 'RW', 'SA', 'SB', 'SC', 'SD',
+                             'SE', 'SG', 'SH', 'SI', 'SJ', 'SK', 'SL', 'SM', 'SN', 'SO', 'SR', 'SS', 'ST', 'SV',
+                             'SX', 'SY', 'SZ', 'TC', 'TD', 'TF', 'TG', 'TH', 'TJ', 'TK', 'TL', 'TM', 'TN', 'TO',
+                             'TR', 'TT', 'TV', 'TW', 'TZ', 'UA', 'UG', 'UM', 'US', 'UY', 'UZ', 'VA', 'VC', 'VE',
+                             'VG', 'VI', 'VN', 'VU', 'WF', 'WS', 'YE', 'YT', 'ZA', 'ZM', 'ZW']:
+        CountryAlias.objects.get_or_create(alias=iso_country_code, country_id=iso_country_code)
+
     CountryAlias.objects.get_or_create(alias="russian federation", country_id="RU")
     CountryAlias.objects.get_or_create(alias="p. r. china", country_id="CN")
     CountryAlias.objects.get_or_create(alias="p.r. china", country_id="CN")
@@ -74,7 +94,6 @@ def add_affiliation_info(apps, schema_editor):
     CountryAlias.objects.get_or_create(alias="grand-duchy of luxembourg", country_id="LU")
     CountryAlias.objects.get_or_create(alias="brasil", country_id="BR")
 
-    
 
 class Migration(migrations.Migration):
 

ietf/stats/tests.py

@@ -3,7 +3,7 @@
 from django.core.urlresolvers import reverse as urlreverse
 
 from ietf.utils.test_data import make_test_data, make_review_data
-from ietf.utils.test_utils import login_testing_unauthorized, TestCase
+from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent
 import ietf.stats.views
 
 class StatisticsTests(TestCase):
@@ -39,6 +39,16 @@ def test_document_stats(self):
                     self.assertTrue(q('#chart'))
                     self.assertTrue(q('table.stats-data'))
 
+    def test_known_country_list(self):
+        make_test_data()
+
+        # check redirect
+        url = urlreverse(ietf.stats.views.known_country_list)
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertTrue("United States" in unicontent(r))
+
     def test_review_stats(self):
         doc = make_test_data()
         review_req = make_review_data(doc)

ietf/stats/urls.py

@@ -6,5 +6,6 @@
 urlpatterns = patterns('',
     url("^$", ietf.stats.views.stats_index),
     url("^document/(?:(?P<stats_type>authors|pages|words|format|formlang|author/documents|author/affiliation|author/country|author/continent|author/citation)/)?$", ietf.stats.views.document_stats),
+    url("^knowncountries/$", ietf.stats.views.known_countries_list),
     url("^review/(?:(?P<stats_type>completion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, ietf.stats.views.review_stats),
 )

ietf/stats/utils.py

@@ -84,20 +84,12 @@ def affiliation_sort_key(affiliation):
     return res
 
 
-
-
 def get_aliased_countries(countries):
     known_aliases = dict(CountryAlias.objects.values_list("alias", "country__name"))
 
-    iso_code_aliases = {}
-
     # add aliases for known countries
     for slug, name in CountryName.objects.values_list("slug", "name"):
-        if len(name) > 2:
-            known_aliases[name.lower()] = name
-
-        if len(slug) == 2 and slug[0].isupper() and slug[1].isupper():
-            iso_code_aliases[slug] = name # add ISO code
+        known_aliases[name.lower()] = name
 
     def lookup_alias(possible_alias):
         name = known_aliases.get(possible_alias)
@@ -185,14 +177,16 @@ def last_text_part_stripped(split):
         if found:
             continue
 
-        # if everything else has failed, try ISO code
-        country = iso_code_aliases.get(country, country)
-        if country in known_countries:
-            res[original_country] = country
-            continue
-
         # unknown country
         res[original_country] = ""
 
     return res
 
+
+def clean_country_name(country_name):
+    if country_name:
+        country_name = get_aliased_countries([country_name]).get(country_name, country_name)
+        if country_name and CountryName.objects.filter(name=country_name).exists():
+            return country_name
+
+    return ""

ietf/stats/views.py

@@ -526,6 +526,18 @@ def count_bins(bins):
     })
 
 
+def known_countries_list(request, stats_type=None, acronym=None):
+    countries = CountryName.objects.prefetch_related("countryalias_set")
+    for c in countries:
+        # the sorting is a bit of a hack - it puts the ISO code first
+        # since it was added in a migration
+        c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk)
+
+    return render(request, "stats/known_countries_list.html", {
+        "countries": countries
+    })
+
+
 @login_required
 def review_stats(request, stats_type=None, acronym=None):
     # This view is a bit complex because we want to show a bunch of

ietf/submit/views.py

@@ -31,6 +31,7 @@
     recently_approved_by_user, validate_submission, create_submission_event,
     docevent_from_submission, post_submission, cancel_submission, rename_submission_files,
     get_person_from_name_email )
+from ietf.stats.utils import clean_country_name
 from ietf.utils.accesstoken import generate_random_key, generate_access_token
 from ietf.utils.draft import Draft
 from ietf.utils.log import log
@@ -401,6 +402,9 @@ def submission_status(request, submission_id, access_token=None):
             # something went wrong, turn this into a GET and let the user deal with it
             return HttpResponseRedirect("")
 
+    for author in submission.authors:
+        author["cleaned_country"] = clean_country_name(author.get("country"))
+
     return render(request, 'submit/submission_status.html', {
         'selected': 'status',
         'submission': submission,

ietf/templates/stats/known_countries_list.html

@@ -0,0 +1,33 @@
+{% extends "base.html" %}
+
+{% load origin %}
+
+{% load ietf_filters staticfiles bootstrap3 %}
+
+{% block content %}
+  {% origin %}
+
+  <h1>{% block title %}Countries known to the Datatracker{% endblock %}</h1>
+
+  <table class="table table-condensed table-striped">
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Aliases (lowercase aliases are matched case-insensitive)</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for c in countries %}
+        <tr>
+          <td>{{ c.name }}</td>
+          <td>
+            {% for a in c.aliases %}
+              {{ a.alias }}{% if not forloop.last %},{% endif %}
+            {% endfor %}
+          </td>
+        </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+
+{% endblock %}

ietf/templates/submit/submission_status.html

@@ -195,9 +195,7 @@ <h2>Meta-data from the submission</h2>
     <tr>
       <th>Authors</th>
       <td>
-        {% with submission.authors as authors %}
-          {{ authors|length }} author{{ authors|pluralize }}
-        {% endwith %}
+        {{ submission.authors|length }} author{{ submission.authors|pluralize }}
         {% if errors.authors %}<p class="text-danger"><b>{{ errors.authors|safe }}</b></p>{% endif %}
       </td>
     </tr>
@@ -207,8 +205,27 @@ <h2>Meta-data from the submission</h2>
         <th>Author {{ forloop.counter }}</th>
         <td>
           {{ author.name }} {% if author.email %}&lt;{{ author.email }}&gt;{% endif %}
-          - {% if author.affiliation %}{{ author.affiliation }}{% else %}<i>unknown affiliation</i>{% endif %}
-          - {% if author.country %}{{ author.country }}{% else %}<i>unknown country</i>{% endif %}
+          -
+          {% if author.affiliation %}
+            {{ author.affiliation }}
+          {% else %}
+            <i>unknown affiliation</i>
+          {% endif %}
+          -
+          {% if author.country %}
+            {{ author.country }}
+            {% if author.cleaned_country and author.country != author.cleaned_country %}
+              (understood to be {{ author.cleaned_country }})
+            {% endif %}
+          {% else %}
+            <i>unknown country</i>
+          {% endif %}
+
+          {% if author.country and not author.cleaned_country %}
+            <br>
+            <b class="text-warning">Unrecognized country: "{{ author.country }}"</b>: Please use a <a href="{% url "ietf.stats.views.known_countries_list" %}">recognized country name</a>.
+          {% endif %}
+          
         </td>
       </tr>
     {% endfor %}