ci: merge main to release (Merge pull request ietf-tools#7089 from ietf-tools/main)

ci: merge main to release

Author: rjsparks

ietf/group/management/commands/import_iesg_appeals.py

@@ -1,5 +1,6 @@
 # Copyright The IETF Trust 2023, All Rights Reserved
 
+import csv
 import datetime
 import re
 import shutil
@@ -148,9 +149,9 @@ def handle(self, *args, **options):
         part_times["klensin-2023-08-15.txt"] = "2023-08-15 15:03:55 -0400"
         part_times["response-to-klensin-2023-08-15.txt"] = "2023-08-24 18:54:13 +0300"
         part_times["hardie-frindell-2023-07-19.txt"] = "2023-07-19 07:17:16PDT"
-        part_times[
-            "response-to-hardie-frindell-2023-07-19.txt"
-        ] = "2023-08-15 11:58:26PDT"
+        part_times["response-to-hardie-frindell-2023-07-19.txt"] = (
+            "2023-08-15 11:58:26PDT"
+        )
         part_times["mcsweeney-2020-07-08.txt"] = "2020-07-08 14:45:00 -0400"
         part_times["response-to-mcsweeney-2020-07-08.pdf"] = "2020-07-28 12:54:04 -0000"
         part_times["gont-2020-04-22.txt"] = "2020-04-22 22:26:20 -0400"
@@ -200,9 +201,9 @@ def handle(self, *args, **options):
         part_times["response-to-sayre-2006-08-29.txt"] = "2006-10-16 13:07:18 -0400"
         part_times["morfin-2006-08-16.pdf"] = "2006-08-16 18:28:19 -0400"
         part_times["response-to-morfin-2006-08-17.txt"] = "2006-08-22 12:05:42 -0400"
-        part_times[
-            "response-to-morfin-2006-08-17-part2.txt"
-        ] = "2006-11-07 13:00:58 -0500"
+        part_times["response-to-morfin-2006-08-17-part2.txt"] = (
+            "2006-11-07 13:00:58 -0500"
+        )
         # part_times["anderson-2006-06-13.txt"]="2006-06-13 21:51:18EDT"
         part_times["anderson-2006-06-13.txt"] = "2006-06-13 21:51:18 -0400"
         part_times["response-to-anderson-2006-06-14.txt"] = "2006-07-10 14:31:08 -0400"
@@ -279,14 +280,15 @@ def handle(self, *args, **options):
                         bits=bits,
                     )
                     redirects.append(
-                        (
-                            part.replace(".md", ".html")
-                            if part.endswith(".md")
-                            else part,
-                            artifact.pk,
-                        )
+                        [
+                            f'www6.ietf.org/iesg/appeal/{part.replace(".md", ".html") if part.endswith(".md") else part}',
+                            f"https://datatracker.ietf.org/group/iesg/appeals/artifact/{artifact.pk}",
+                            302,
+                        ]
                     )
 
         shutil.rmtree(tmpdir)
-        with open("iesg_appeal_redirects.txt", "w") as f:
-            f.write(str(redirects))
+        with open("iesg_appeal_redirects.csv", "w", newline="") as f:
+            csvwriter = csv.writer(f)
+            for row in redirects:
+                csvwriter.writerow(row)

ietf/group/management/commands/import_iesg_statements.py

@@ -2,6 +2,7 @@
 
 import debug  # pyflakes:ignore
 
+import csv
 import datetime
 import os
 import shutil
@@ -41,8 +42,11 @@ def handle(self, *args, **options):
             self.stdout.write(f"Clean up {tmpdir} manually")
             exit(-1)
 
+        redirects = []
         for item in self.get_work_items():
-            replaced = item.title.endswith(" SUPERSEDED") or item.doc_time.date() == datetime.date(2007,7,30)
+            replaced = item.title.endswith(
+                " SUPERSEDED"
+            ) or item.doc_time.date() == datetime.date(2007, 7, 30)
             title = item.title
             if title.endswith(" - SUPERSEDED"):
                 title = title[: -len(" - SUPERSEDED")]
@@ -93,20 +97,34 @@ def handle(self, *args, **options):
                 os.makedirs(dest.parent, exist_ok=True)
                 shutil.copy(source, dest)
 
+            redirects.append(
+                [
+                    f"www.ietf.org/about/groups/iesg/statements/{item.slug}",
+                    f"https://datatracker.ietf.org/group/iesg/statements/{name}",
+                    302,
+                ]
+            )
+
         shutil.rmtree(tmpdir)
+        with open("iesg_statement_redirects.csv", "w", newline="") as f:
+            csvwriter = csv.writer(f)
+            for row in redirects:
+                csvwriter.writerow(row)
 
     def get_work_items(self):
-        Item = namedtuple("Item", "doc_time source_filename title")
+        Item = namedtuple("Item", "doc_time source_filename title slug")
         items = []
         dressed_rows = " ".join(
             self.cut_paste_from_www().expandtabs(1).split(" ")
         ).split("\n")
+        old_slugs = self.get_old_slugs()
         # Rube-Goldberg-esque dance to deal with conflicting directions of the scrape and
         # what order we want the result to sort to
         dressed_rows.reverse()
+        old_slugs.reverse()
         total_times_date_seen = Counter([row.split(" ")[0] for row in dressed_rows])
         count_date_seen_so_far = Counter()
-        for row in dressed_rows:
+        for row, slug in zip(dressed_rows, old_slugs):
             date_part = row.split(" ")[0]
             title_part = row[len(date_part) + 1 :]
             datetime_args = list(map(int, date_part.replace("-0", "-").split("-")))
@@ -120,6 +138,7 @@ def get_work_items(self):
                     doc_time,
                     f"{date_part}-{total_times_date_seen[date_part] - count_date_seen_so_far[date_part]}.md",
                     title_part,
+                    slug,
                 )
             )
         return items
@@ -187,3 +206,69 @@ def cut_paste_from_www(self):
 2000-11-20	A New IETF Work Area
 2000-08-29	Guidance on Interim IETF Working Group Meetings and Conference Calls - SUPERSEDED
 2000-08-29	IESG Guidance on the Moderation of IETF Working Group Mailing Lists"""
+
+    def get_old_slugs(self):
+        return [
+            "support-documents",
+            "interim-meetings-guidance",
+            "ethertypes",
+            "second-report-on-the-rfc-8989-experiment",
+            "interim-meetings-guidance-2023-01-27",
+            "statement-on-restricting-access",
+            "handling-ballot-positions",
+            "report-on-rfc8989-experiment",
+            "email-addresses-ietf-domain",
+            "on-inclusive-language",
+            "internet-draft-authorship",
+            "processing-errata-ietf-stream",
+            "last-call-guidance",
+            "statement-on-oppressive-exclusionary-language",
+            "interim-meetings-guidance-2020-05-01",
+            "meeting-photography-policy",
+            "interim-meetings-guidance-2018-01-11",
+            "open-source-repositories-license",
+            "support-documents-2016-11-13",
+            "interim-meetings-guidance-2016-02-05",
+            "interim-meetings-guidance-2016-01-11",
+            "maximizing-encrypted-access",
+            "internet-draft-authorship-2015-06-11",
+            "designating-rfcs-historic",
+            "iesg-discuss-criteria",
+            "writable-mib-module",
+            "anti-harassment-policy",
+            "ethertypes-2012-10-25",
+            "internet-draft-removal",
+            "designating-rfcs-historic-2011-10-20",
+            "designating-rfcs-historic-2011-06-27",
+            "rfc-metadata-errata",
+            "document-shepherds",
+            "assignable-codepoints-addresses-names",
+            "nomcom-eligibility-day-passes",
+            "copyright-2009-09-08",
+            "reserving-resources-examples",
+            "interim-meetings-guidance-2008-09-02",
+            "processing-rfc-errata",
+            "spam-control-2008-04-14",
+            "registration-requests-uris",
+            "urn-namespaces-registry",
+            "off-topic-postings",
+            "appeals-actions-decisions",
+            "experimental-congestion-control",
+            "area-director-sponsoring-documents",
+            "last-call-guidance-2007-01-15",
+            "normative-informative-references",
+            "disruptive-posting",
+            "spam-control-2006-01-09",
+            "auth48",
+            "syntax-format-definitions",
+            "idn",
+            "copyright-2002-11-27",
+            "spam-control-2002-03-13",
+            "design-teams",
+            "formal-languages-use",
+            "sub-ip-area-2001-03-21",
+            "sub-ip-area-2000-11-20",
+            "sub-ip-area-2000-12-06",
+            "interim-meetings-guidance-2000-08-29",
+            "mailing-lists-moderation",
+        ]

ietf/group/views.py

@@ -2209,7 +2209,6 @@ def statements(request, acronym, group_type=None):
         )
         .order_by("-published")
     )
-    debug.show("statements.first().status")
     return render(
         request,
         "group/statements.html",

ietf/meeting/management/commands/import_iesg_minutes.py

@@ -1,6 +1,7 @@
 # Copyright The IETF Trust 2023, All Rights Reserved
 
 from collections import namedtuple
+import csv
 import datetime
 import os
 import re
@@ -158,6 +159,7 @@ def handle(self, *args, **options):
         minutes_dir = Path(old_minutes_root)
         date_re = re.compile(r"\d{4}-\d{2}-\d{2}")
         meeting_times = set()
+        redirects = []
         for file_prefix in ["minutes", "narrative"]:
             paths = list(minutes_dir.glob(f"[12][09][0129][0-9]/{file_prefix}*.txt"))
             paths.extend(
@@ -259,6 +261,13 @@ def handle(self, *args, **options):
                     else:
                         os.makedirs(dest.parent, exist_ok=True)
                         shutil.copy(source, dest)
+                    redirects.append(
+                        [
+                            f"www6.ietf.org/iesg/minutes/{dt.year}/{bof_coord_data[dt].source_name}",
+                            f"https://datatracker.ietf.org/doc/{doc_name}",
+                            302,
+                        ]
+                    )
             else:
                 for type_id in ["minutes", "narrativeminutes"]:
                     source_file_prefix = (
@@ -334,10 +343,22 @@ def handle(self, *args, **options):
                                 html_content = re.sub(
                                     '<a href="http://validator.w3.org/[^>]*> *<img[^>]*></a>',
                                     "",
-                                    html_content
+                                    html_content,
                                 )
                                 dest.write_text(html_content, encoding="utf-8")
                             else:
                                 shutil.copy(txt_source, dest)
+                            redirects.append(
+                                [
+                                    f"www6.ietf.org/iesg/minutes/{dt.year}/{txt_source.name if txt_source.exists() else html_source.name}",
+                                    f"https://datatracker.ietf.org/doc/{doc_name}",
+                                    302,
+                                ]
+                            )
 
             counter += 1
+
+        with open("iesg_minutes_redirects.csv", "w", newline="") as f:
+            csvwriter = csv.writer(f)
+            for row in redirects:
+                csvwriter.writerow(row)