[pull] main from ietf-tools:main

ietf/doc/tests_utils_rfc_json.py

@@ -14,6 +14,7 @@
     RfcAuthorFactory,
     RfcFactory,
     RgRfcFactory,
+    WgDraftFactory,
     WgRfcFactory,
 )
 from ietf.doc.models import RelatedDocument
@@ -199,6 +200,19 @@ def test_errata_url_set_when_errata_exist(self):
             f"https://www.rfc-editor.org/errata/rfc{rfc.rfc_number}",
         )
 
+    def test_draft_field_includes_revision(self):
+        """draft field is '<name>-<rev>' when the RFC originated from a draft."""
+        draft = WgDraftFactory(rev="07")
+        rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc
+        draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc)
+        _put_pub_levels(rfc.rfc_number, "ps")
+        _put_empty_errata()
+
+        generate_rfc_json(rfc.rfc_number)
+        data = _read_json(rfc.rfc_number)
+
+        self.assertEqual(data["draft"], f"{draft.name}-07")
+
     def test_errata_url_none_when_no_errata(self):
         """errata_url is None when errata.json has no entries for the RFC."""
         rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc

ietf/doc/utils_rfc_json.py

@@ -55,7 +55,7 @@ def generate_rfc_json(rfc_number: int, *, pub_levels=None) -> None:
 
     # draft name
     draft_doc = rfc.came_from_draft()
-    draft = draft_doc.name if draft_doc else None
+    draft = f"{draft_doc.name}-{draft_doc.rev}" if draft_doc else None
 
     # authors: ordered list of display strings
     authors = []

ietf/group/tests_info.py

@@ -27,7 +27,7 @@
 
 from ietf.community.models import CommunityList
 from ietf.community.utils import reset_name_contains_index_for_rule
-from ietf.doc.factories import WgDraftFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory
+from ietf.doc.factories import WgDraftFactory, WgRfcFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory
 from ietf.doc.models import Document, DocEvent, State
 from ietf.doc.storage_utils import retrieve_str
 from ietf.doc.utils_charter import charter_name_for_group
@@ -396,6 +396,7 @@ def test_group_documents(self):
         draft7 = WgDraftFactory(group=group)
         draft7.set_state(State.objects.get(type='draft', slug='expired'))
         draft7.set_state(State.objects.get(type='draft-stream-%s' % draft7.stream_id, slug='dead')) # Expired WG draft, marked as dead
+        rfc = WgRfcFactory(group=group)
 
         clist = CommunityList.objects.get(group=group)
         related_docs_rule = clist.searchrule_set.get(rule_type='name_contains')
@@ -426,6 +427,12 @@ def test_group_documents(self):
         q = PyQuery(r.content)
         self.assertTrue(any([draft2.name in x.attrib['href'] for x in q('table td a.track-untrack-doc')]))
 
+        # RFC rows must use the RFC number as the sort key so that numeric sort
+        # is not disrupted by the page-count text that precedes the name in the cell.
+        # Draft rows must use the document name.
+        self.assertTrue(q(f'td.doc[data-sort-number="{rfc.rfc_number}"]'))
+        self.assertTrue(q(f'td.doc[data-sort-number="{draft.name}"]'))
+
         # Let's also check the IRTF stream
         rg = GroupFactory(type_id='rg')
         setup_default_community_list_for_group(rg)

ietf/nomcom/tests.py

@@ -1640,6 +1640,20 @@ def test_feedback_topic_badges(self):
         q = PyQuery(response.content)
         self.assertEqual( len(q('.text-bg-success')), 0 )
 
+    def test_feedback_index_sort_keys(self):
+        url = reverse('ietf.nomcom.views.view_feedback', kwargs={'year': self.nc.year()})
+        login_testing_unauthorized(self, self.member.user.username, url)
+        provide_private_key_to_test_client(self)
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        q = PyQuery(response.content)
+        # Feedback count cells must carry a numeric data-sort-number so that
+        # a "New" badge appearing before the count doesn't corrupt the sort key.
+        sort_cells = q('td[data-sort-number]')
+        self.assertTrue(len(sort_cells) > 0)
+        for cell in sort_cells.items():
+            self.assertRegex(cell.attr('data-sort-number'), r'^\d+$')
+
 class NewActiveNomComTests(TestCase):
 
     def setUp(self):

ietf/settings.py

@@ -977,6 +977,10 @@ def skip_unreadable_post(record):
 )
 RFC_FILE_TYPES = IDSUBMIT_FILE_TYPES
 
+# Paths in the red bucket
+RFCINDEX_INPUT_PATH = "other/"
+RFCINDEX_OUTPUT_PATH = "other/"
+
 IDSUBMIT_MAX_DRAFT_SIZE =  {
     'txt':  2*1024*1024,  # Max size of txt draft file in bytes
     'xml':  3*1024*1024,  # Max size of xml draft file in bytes

ietf/sync/rfcindex.py

@@ -48,9 +48,17 @@ def errata_url(rfc: Document):
     return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}")
 
 
+def red_bucket_input_path(filename: str) -> str:
+    return str(Path(settings.RFCINDEX_INPUT_PATH) / filename)
+
+
+def red_bucket_output_path(filename: str) -> str:
+    return str(Path(settings.RFCINDEX_OUTPUT_PATH) / filename)
+
+
 def save_to_red_bucket(filename: str, content: str | bytes):
     red_bucket = storages["red_bucket"]
-    bucket_path = str(Path(getattr(settings, "RFCINDEX_OUTPUT_PATH", "")) / filename)
+    bucket_path = red_bucket_output_path(filename)
     if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True):
         # Django 4.2's FileSystemStorage does not support allow_overwrite.
         red_bucket.delete(bucket_path)
@@ -87,8 +95,7 @@ class UnusableRfcNumber:
 
 
 def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]:
-    FILENAME = "unusable-rfc-numbers.json"
-    bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME)
+    bucket_path = red_bucket_input_path("unusable-rfc-numbers.json") 
     try:
         with storages["red_bucket"].open(bucket_path) as urn_file:
             records = json.load(urn_file)
@@ -115,8 +122,7 @@ def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]:
 
 
 def get_april1_rfc_numbers() -> Container[int]:
-    FILENAME = "april-first-rfc-numbers.json"
-    bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME)
+    bucket_path = red_bucket_input_path("april-first-rfc-numbers.json")
     try:
         with storages["red_bucket"].open(bucket_path) as urn_file:
             records = json.load(urn_file)
@@ -139,8 +145,7 @@ def get_april1_rfc_numbers() -> Container[int]:
 
 
 def get_publication_std_levels() -> dict[int, StdLevelName]:
-    FILENAME = "publication-std-levels.json"
-    bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME)
+    bucket_path = red_bucket_input_path("publication-std-levels.json")
     values: dict[int, StdLevelName] = {}
     try:
         with storages["red_bucket"].open(bucket_path) as urn_file:

ietf/sync/tests_rfcindex.py

@@ -28,9 +28,11 @@
     get_april1_rfc_numbers,
     get_publication_std_levels,
     get_unusable_rfc_numbers,
+    red_bucket_input_path,
+    red_bucket_output_path,
+    save_to_filesystem,
     save_to_red_bucket,
     subseries_text_line,
-    save_to_filesystem,
 )
 from ietf.utils.test_utils import TestCase
 
@@ -401,25 +403,47 @@ def test_create_fyi_txt_index(self, mock_save_blob, mock_save_file):
         )
 
 
+@override_settings(RFCINDEX_INPUT_PATH="input/", RFCINDEX_OUTPUT_PATH="output/")
 class HelperTests(TestCase):
+    INPUT_PATH = "input"
+    OUTPUT_PATH = "output"
+
     def test_format_rfc_number(self):
         self.assertEqual(format_rfc_number(10), "10")
         with override_settings(RFCINDEX_MATCH_LEGACY_XML=True):
             self.assertEqual(format_rfc_number(10), "0010")
 
+    def test_red_bucket_input_path(self):
+        with override_settings(RFCINDEX_INPUT_PATH="bar"):
+            self.assertEqual(red_bucket_input_path("foo"), "bar/foo")
+        with override_settings(RFCINDEX_INPUT_PATH="bar/"):
+            self.assertEqual(red_bucket_input_path("foo"), "bar/foo")
+
+    def test_red_bucket_output_path(self):
+        self.assertEqual(red_bucket_input_path("foo"), f"{self.INPUT_PATH}/foo")
+        with override_settings(RFCINDEX_OUTPUT_PATH="bar"):
+            self.assertEqual(red_bucket_output_path("foo"), "bar/foo")
+        with override_settings(RFCINDEX_OUTPUT_PATH="bar/"):
+            self.assertEqual(red_bucket_output_path("foo"), "bar/foo")
+
     def test_save_to_red_bucket(self):
         red_bucket = storages["red_bucket"]
         with override_settings(RFCINDEX_DELETE_THEN_WRITE=False):
             save_to_red_bucket("test", "contents \U0001f600")
         # Read as binary and explicitly decode to confirm encoding
-        with red_bucket.open("test", "rb") as f:
+        with red_bucket.open(f"{self.OUTPUT_PATH}/test", "rb") as f:
             self.assertEqual(f.read().decode("utf-8"), "contents \U0001f600")
         with override_settings(RFCINDEX_DELETE_THEN_WRITE=True):
             save_to_red_bucket("test", "new contents \U0001fae0".encode("utf-8"))
         # Read as binary and explicitly decode to confirm encoding
-        with red_bucket.open("test", "rb") as f:
+        with red_bucket.open(f"{self.OUTPUT_PATH}/test", "rb") as f:
             self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0")
-        red_bucket.delete("test")  # clean up like a good child
+        red_bucket.delete(f"{self.OUTPUT_PATH}/test")  # clean up like a good child
+        # check that we can override the path
+        with override_settings(RFCINDEX_OUTPUT_PATH="fruit"):
+            save_to_red_bucket("test", "content")
+        self.assertTrue(red_bucket.exists("fruit/test"))
+        red_bucket.delete("fruit/test")  # clean up like a good child
 
     def test_save_to_filesystem(self):
         rfc_path = Path(settings.RFC_PATH)
@@ -445,30 +469,36 @@ def test_get_unusable_rfc_numbers_raises(self):
         with self.assertRaises(FileNotFoundError):
             get_unusable_rfc_numbers()
         red_bucket = storages["red_bucket"]
-        red_bucket.save("unusable-rfc-numbers.json", ContentFile("not json"))
+        red_bucket.save(
+            f"{self.INPUT_PATH}/unusable-rfc-numbers.json", ContentFile("not json")
+        )
         with self.assertRaises(json.JSONDecodeError):
             get_unusable_rfc_numbers()
-        red_bucket.delete("unusable-rfc-numbers.json")
+        red_bucket.delete(f"{self.INPUT_PATH}/unusable-rfc-numbers.json")
 
     def test_get_april1_rfc_numbers_raises(self):
         """get_april1_rfc_numbers should bail on errors"""
         with self.assertRaises(FileNotFoundError):
             get_april1_rfc_numbers()
         red_bucket = storages["red_bucket"]
-        red_bucket.save("april-first-rfc-numbers.json", ContentFile("not json"))
+        red_bucket.save(
+            f"{self.INPUT_PATH}/april-first-rfc-numbers.json", ContentFile("not json")
+        )
         with self.assertRaises(json.JSONDecodeError):
             get_april1_rfc_numbers()
-        red_bucket.delete("april-first-rfc-numbers.json")
+        red_bucket.delete(f"{self.INPUT_PATH}/april-first-rfc-numbers.json")
 
     def test_get_publication_std_levels_raises(self):
         """get_publication_std_levels should bail on errors"""
         with self.assertRaises(FileNotFoundError):
             get_publication_std_levels()
         red_bucket = storages["red_bucket"]
-        red_bucket.save("publication-std-levels.json", ContentFile("not json"))
+        red_bucket.save(
+            f"{self.INPUT_PATH}/publication-std-levels.json", ContentFile("not json")
+        )
         with self.assertRaises(json.JSONDecodeError):
             get_publication_std_levels()
-        red_bucket.delete("publication-std-levels.json")
+        red_bucket.delete(f"{self.INPUT_PATH}/publication-std-levels.json")
 
     def test_subseries_text_line(self):
         text = "foobar"

ietf/templates/doc/search/search_result_row.html

@@ -45,7 +45,7 @@
             </a>
         {% endfor %}
     </td>
-    <td class="doc bg-transparent">
+    <td class="doc bg-transparent" data-sort-number="{% if doc.type_id == "rfc" %}{{ doc.rfc_number }}{% else %}{{ doc.name }}{% endif %}">
         {% if doc.pages %}<small class="float-end text-body-secondary d-none d-sm-block">{{ doc.pages }} page{{ doc.pages|pluralize }}</small>{% endif %}
         <div>
             <a href="{{ doc.get_absolute_url }}">

ietf/templates/nomcom/view_feedback.html

@@ -43,7 +43,7 @@ <h3 class="mt-5" id="declined">Declined each nominated position</h3>
                             </a>
                         </td>
                         {% for fbtype_name, fbtype_count, fbtype_newflag in fb_dict.feedback %}
-                            <td>
+                            <td data-sort-number="{{ fbtype_count }}">
                                 {% if fbtype_newflag %}<span class="badge rounded-pill text-bg-success">New</span>{% endif %}
                                 {{ fbtype_count }}
                             </td>
@@ -82,7 +82,7 @@ <h2 class="mt-5" id="topics">Feedback related to topics</h2>
                         </a>
                     </td>
                     {% for fbtype_name, fbtype_count, fbtype_newflag in fb_dict.feedback %}
-                        <td>
+                        <td data-sort-number="{{ fbtype_count }}">
                             {% if fbtype_newflag %}<span class="badge rounded-pill text-bg-success">New</span>{% endif %}
                             {{ fbtype_count }}
                         </td>

k8s/settings_local.py

@@ -472,8 +472,10 @@ def _multiline_to_list(s):
     ),
 }
 RFCINDEX_DELETE_THEN_WRITE = False  # S3Storage allows file_overwrite by default
-RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/")
-RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH", "")
+if "DATATRACKER_RFCINDEX_OUTPUT_PATH" in os.environ:
+    RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH")
+if "DATATRACKER_RFCINDEX_INPUT_PATH" in os.environ:
+    RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH")
 
 # Configure the blobdb app for artifact storage
 _blobdb_replication_enabled = (