fix: AD order in IESG ballots (and other places) does not handle UTF-8 collation

ietf/doc/views_doc.py

@@ -91,7 +91,7 @@
 from ietf.utils.response import permission_denied
 from ietf.utils.text import maybe_split
 from ietf.utils.timezone import date_today
-
+from ietf.utils.unicodenormalize import normalize_for_sorting
 
 def render_document_top(request, doc, tab, name):
     tabs = []
@@ -1500,7 +1500,7 @@ def document_ballot_content(request, doc, ballot_id, editable=True):
     position_groups = []
     for n in BallotPositionName.objects.filter(slug__in=[p.pos_id for p in positions]).order_by('order'):
         g = (n, [p for p in positions if p.pos_id == n.slug])
-        g[1].sort(key=lambda p: (p.is_old_pos, p.balloter.plain_name()))
+        g[1].sort(key=lambda p: (p.is_old_pos, normalize_for_sorting(p.balloter.plain_name())))
         if n.blocking:
             position_groups.insert(0, g)
         else:

ietf/iesg/tests.py

@@ -281,12 +281,13 @@ def test_working_groups(self):
             wg_summary,
         ) = get_wg_dashboard_info()
 
+        # checks for the expected result with area sorted by name
         self.assertEqual(
             area_summary,
             [
                 {
-                    "area": "red",
-                    "groups_in_area": 1,
+                    "area": "blue",
+                    "groups_in_area": 3,
                     "groups_with_docs": 0,
                     "doc_count": 0,
                     "page_count": 0,
@@ -295,8 +296,8 @@ def test_working_groups(self):
                     "page_percent": 0,
                 },
                 {
-                    "area": "orange",
-                    "groups_in_area": 4,
+                    "area": "green",
+                    "groups_in_area": 3,
                     "groups_with_docs": 0,
                     "doc_count": 0,
                     "page_count": 0,
@@ -305,8 +306,8 @@ def test_working_groups(self):
                     "page_percent": 0,
                 },
                 {
-                    "area": "yellow",
-                    "groups_in_area": 2,
+                    "area": "orange",
+                    "groups_in_area": 4,
                     "groups_with_docs": 0,
                     "doc_count": 0,
                     "page_count": 0,
@@ -315,8 +316,8 @@ def test_working_groups(self):
                     "page_percent": 0,
                 },
                 {
-                    "area": "green",
-                    "groups_in_area": 3,
+                    "area": "red",
+                    "groups_in_area": 1,
                     "groups_with_docs": 0,
                     "doc_count": 0,
                     "page_count": 0,
@@ -325,8 +326,8 @@ def test_working_groups(self):
                     "page_percent": 0,
                 },
                 {
-                    "area": "blue",
-                    "groups_in_area": 3,
+                    "area": "violet",
+                    "groups_in_area": 4,
                     "groups_with_docs": 0,
                     "doc_count": 0,
                     "page_count": 0,
@@ -335,8 +336,8 @@ def test_working_groups(self):
                     "page_percent": 0,
                 },
                 {
-                    "area": "violet",
-                    "groups_in_area": 4,
+                    "area": "yellow",
+                    "groups_in_area": 2,
                     "groups_with_docs": 0,
                     "doc_count": 0,
                     "page_count": 0,
@@ -1192,34 +1193,14 @@ def test_working_groups(self):
             area_summary,
             [
                 {
-                    "area": "red",
-                    "groups_in_area": 1,
-                    "groups_with_docs": 1,
-                    "doc_count": 1,
-                    "page_count": 7,
-                    "group_percent": 6.25,
-                    "doc_percent": 3.571428571428571,
-                    "page_percent": 3.5897435897435894,
-                },
-                {
-                    "area": "orange",
-                    "groups_in_area": 4,
+                    "area": "blue",
+                    "groups_in_area": 3,
                     "groups_with_docs": 3,
-                    "doc_count": 4,
-                    "page_count": 29,
+                    "doc_count": 6,
+                    "page_count": 40,
                     "group_percent": 18.75,
-                    "doc_percent": 14.285714285714285,
-                    "page_percent": 14.871794871794872,
-                },
-                {
-                    "area": "yellow",
-                    "groups_in_area": 2,
-                    "groups_with_docs": 2,
-                    "doc_count": 2,
-                    "page_count": 17,
-                    "group_percent": 12.5,
-                    "doc_percent": 7.142857142857142,
-                    "page_percent": 8.717948717948717,
+                    "doc_percent": 21.428571428571427,
+                    "page_percent": 20.51282051282051,
                 },
                 {
                     "area": "green",
@@ -1232,14 +1213,24 @@ def test_working_groups(self):
                     "page_percent": 11.282051282051283,
                 },
                 {
-                    "area": "blue",
-                    "groups_in_area": 3,
+                    "area": "orange",
+                    "groups_in_area": 4,
                     "groups_with_docs": 3,
-                    "doc_count": 6,
-                    "page_count": 40,
+                    "doc_count": 4,
+                    "page_count": 29,
                     "group_percent": 18.75,
-                    "doc_percent": 21.428571428571427,
-                    "page_percent": 20.51282051282051,
+                    "doc_percent": 14.285714285714285,
+                    "page_percent": 14.871794871794872,
+                },
+                {
+                    "area": "red",
+                    "groups_in_area": 1,
+                    "groups_with_docs": 1,
+                    "doc_count": 1,
+                    "page_count": 7,
+                    "group_percent": 6.25,
+                    "doc_percent": 3.571428571428571,
+                    "page_percent": 3.5897435897435894,
                 },
                 {
                     "area": "violet",
@@ -1251,6 +1242,16 @@ def test_working_groups(self):
                     "doc_percent": 35.714285714285715,
                     "page_percent": 41.02564102564102,
                 },
+                {
+                    "area": "yellow",
+                    "groups_in_area": 2,
+                    "groups_with_docs": 2,
+                    "doc_count": 2,
+                    "page_count": 17,
+                    "group_percent": 12.5,
+                    "doc_percent": 7.142857142857142,
+                    "page_percent": 8.717948717948717,
+                },
             ],
         )
         self.assertEqual(

ietf/iesg/utils.py

@@ -12,7 +12,7 @@
 from ietf.group.models import Group
 from ietf.iesg.agenda import get_doc_section
 from ietf.person.utils import get_active_ads
-
+from ietf.utils.unicodenormalize import normalize_for_sorting
 
 TelechatPageCount = namedtuple(
     "TelechatPageCount",
@@ -192,6 +192,7 @@ def get_wg_dashboard_info():
                 else 0,
             }
         )
+    area_summary.sort(key=lambda r: r["area"])
     area_totals = {
         "group_count": groups_total,
         "doc_count": docs_total,
@@ -238,7 +239,7 @@ def get_wg_dashboard_info():
                     else 0,
                 }
             )
-    noad_summary.sort(key=lambda r: (r["ad"], r["area"]))
+    noad_summary.sort(key=lambda r: (normalize_for_sorting(r["ad"]), r["area"]))
 
     ad_summary = []
     ad_totals = {
@@ -278,7 +279,7 @@ def get_wg_dashboard_info():
                     else 0,
                 }
             )
-    ad_summary.sort(key=lambda r: (r["ad"], r["area"]))
+    ad_summary.sort(key=lambda r: (normalize_for_sorting(r["ad"]), r["area"]))
 
     rfc_counter = Counter(
         Document.objects.filter(type="rfc").values_list("group__acronym", flat=True)

ietf/iesg/views.py

@@ -68,6 +68,7 @@
 from ietf.meeting.utils import get_activity_stats
 from ietf.doc.utils_search import fill_in_document_table_attributes, fill_in_telechat_date
 from ietf.utils.timezone import date_today, datetime_from_date
+from ietf.utils.unicodenormalize import normalize_for_sorting
 
 def review_decisions(request, year=None):
     events = DocEvent.objects.filter(type__in=("iesg_disapproved", "iesg_approved"))
@@ -547,7 +548,7 @@ def milestones_needing_review(request):
             )
 
     return render(request, 'iesg/milestones_needing_review.html',
-                  dict(ads=sorted(ad_list, key=lambda ad: ad.plain_name()),))
+                  dict(ads=sorted(ad_list, key=lambda ad: normalize_for_sorting(ad.plain_name())),))
 
 def photos(request):
     roles = sorted(Role.objects.filter(group__type='area', group__state='active', name_id='ad'),key=lambda x: "" if x.group.acronym=="gen" else x.group.acronym)

ietf/utils/unicodenormalize.py

@@ -0,0 +1,9 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+import unicodedata
+
+def normalize_for_sorting(text):
+    """Normalize text for proper accent-aware sorting."""
+    # Normalize the text to NFD (decomposed form)
+    decomposed = unicodedata.normalize('NFD', text)
+    # Filter out combining diacritical marks
+    return ''.join(char for char in decomposed if not unicodedata.combining(char))