From 5df4bf9ddc25253c25a350a2be44c6458d3e9cfd Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 21 Aug 2026 13:23:04 -0500 Subject: [PATCH] fix: generate idnits2 rfc-status blob for 5-digit RFC numbers generate_idnits2_rfc_status() allocated a fixed 10000-element array and indexed it by RFC number, so it raised IndexError for any RFC above 10000. The task has been failing on every run since 2026-06-16, and because ietf/doc/tasks.py computes the blob outside its try block, the exception escapes before anything is written. The served file has been frozen at 9998 characters since then (content-length 10154), stale for all RFCs rather than only 5-digit ones. This commit sizes the array from the highest rfc_number instead. The most recent versions of idnits2 (through 2.17.1) will correctly consume this larger array without modification. It also stops the generator crashing on RFC rows it doesn't expect by excluding RFCs with a null rfc_number (int(None) raises TypeError) and falling back to 'U' for an unrecognised std_level_id (symbols[None] raises KeyError). Document.std_level is nullable, and a single such row would take down the whole task. To allow existing idnits clients at version 2.17.1 and below to keep operating, override RFC 16 to 'O'. This deliberately contradicts both the datatracker and the RFC Editor, which record RFC 16 as updated rather than obsoleted. idnits2 validates its download of this file by matching the first 64 characters against a literal pattern asserting 'O' at position 16. The reason is lost, but it was likely the result of manual curation at tools.ietf.org long ago. Without the override, existing idnits2 clients discard the file as corrupt, fall back to whatever stale copy they have, and silently perform no RFC status checks at all. This is independent of the crash and predates it. Note that the generator uses a floor of 6312. This is required because the RFC 16, RFC 200 and RFC 6312 workarounds write those offsets unconditionally, so the array must reach 6312 regardless of the data; without the floor the generator raises IndexError for any dataset whose highest RFC is below that. It also keeps output identical to the previous behaviour, where the fixed 10000-element array always had those offsets in range. Making the workaround writes conditional instead would remove the need for the floor, but that was not done here. Verification: - Against the production snapshot, positions 1..9993 and the first line are byte-identical to the pre-change algorithm; the blob extends from 9999 to 10031. rfc10001='B', rfc10008='P', rfc10031='P' match their std_level_id values. - idnits2's own download validation (grep -qsE against the first line) now passes, where it fails against the file production serves today. - idnits2's lookup path resolves 5-digit statuses correctly against the generated file: rfc10001 -> Best Current Practice, rfc10031 -> Proposed Standard, rfc10032 -> past end of blob. - ietf.doc.tests (122 tests) and ietf.doc.tests_tasks ietf.doc.tests_downref (15 tests) pass. This commit produced primarily by Claude. --- ietf/doc/tests.py | 15 +++++++++++++++ ietf/doc/utils.py | 26 +++++++++++++++++++++----- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 1947fe272a..83a152c148 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -3648,8 +3648,23 @@ def test_obsoleted(self): def test_generate_idnits2_rfc_status(self): for slug in ('bcp', 'ds', 'exp', 'hist', 'inf', 'std', 'ps', 'unkn'): WgRfcFactory(std_level_id=slug) + WgRfcFactory(rfc_number=10001, std_level_id='ps') + WgRfcFactory(rfc_number=10002, std_level_id=None) blob = generate_idnits2_rfc_status().replace("\n", "") self.assertEqual(blob[6312-1], "O") + # idnits2 discards the whole file if this is not "O" - see generate_idnits2_rfc_status + self.assertEqual(blob[16-1], "O") + self.assertEqual(blob[10001-1], "P") + self.assertEqual(blob[10002-1], "U") + + def test_generate_idnits2_rfc_status_low_numbers_only(self): + # The workarounds write fixed offsets, so the blob has to reach 6312 even when + # no RFC does. + WgRfcFactory(rfc_number=1001, std_level_id="ps") + blob = generate_idnits2_rfc_status().replace("\n", "") + self.assertEqual(blob[6312-1], "O") + self.assertEqual(blob[200-1], "O") + self.assertEqual(blob[16-1], "O") def test_rfc_status(self): url = urlreverse('ietf.doc.views_doc.idnits2_rfc_status') diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index a353cfa3f2..301c352a0d 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -21,7 +21,7 @@ from django.conf import settings from django.contrib import messages from django.core.cache import caches -from django.db.models import OuterRef +from django.db.models import Max, OuterRef from django.forms import ValidationError from django.http import Http404 from django.template.loader import render_to_string @@ -1386,8 +1386,6 @@ def update_doc_extresources(doc, new_resources, by): def generate_idnits2_rfc_status(): - blob=['N']*10000 - symbols={ 'ps': 'P', 'inf': 'I', @@ -1399,10 +1397,17 @@ def generate_idnits2_rfc_status(): 'unkn': 'U', } - rfcs = Document.objects.filter(type_id='rfc') + rfcs = Document.objects.filter(type_id='rfc').exclude(rfc_number=None) + + # One character per RFC number, indexed by that number, so the array has to reach the + # highest RFC published. The floor keeps the fixed offsets in the workarounds below in + # range when only a few RFCs exist, as is the case under test. + highest = rfcs.aggregate(Max('rfc_number'))['rfc_number__max'] or 0 + blob=['N']*max(highest, 6312) + for rfc in rfcs: offset = int(rfc.rfc_number)-1 - blob[offset] = symbols[rfc.std_level_id] + blob[offset] = symbols.get(rfc.std_level_id, 'U') if rfc.related_that('obs'): blob[offset] = 'O' @@ -1422,6 +1427,17 @@ def generate_idnits2_rfc_status(): # RFC200 is an old RFC List by Number blob[200 -1] = 'O' + # !! Do not remove: idnits2 rejects this entire file if RFC16 is not 'O' !! + # + # This deliberately contradicts both the datatracker and the RFC Editor, which + # record RFC16 as updated rather than obsoleted. idnits2 validates its download + # of this file by matching the first 64 characters against a literal pattern that + # asserts 'O' here, inherited from a tools.ietf.org curation that disagreed with + # the RFC Editor. A mismatch makes idnits2 discard the file as corrupt and fall + # back to whatever stale copy it has, silently performing no RFC status checks at + # all. Removing this line therefore breaks every idnits2 client, not just RFC16. + blob[16 - 1] = 'O' + # End Workarounds blob = re.sub('N*$','',''.join(blob))