diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 7393e83b829..5bb26e4f6fa 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -17,6 +17,6 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@v7 - name: 'Dependency Review' - uses: actions/dependency-review-action@v4 + uses: actions/dependency-review-action@v5 with: vulnerability-check: false diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml index 10d04c8265e..852e2d6fedd 100644 --- a/.github/workflows/reusable-tests.yml +++ b/.github/workflows/reusable-tests.yml @@ -75,7 +75,7 @@ jobs: path: geckodriver.log - name: Upload Coverage Results to Codecov - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7 with: disable_search: true files: coverage.xml diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 72bdb7a7ae1..aa1ab4a55cf 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260605T2314 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260711T0341 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 7ca47fd1979..2fe337efb27 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260605T2314 +20260711T0341 diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index 00733717c4f..f4bfe8330ee 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -10,6 +10,7 @@ from django.apps import apps as django_apps from django.core.exceptions import ObjectDoesNotExist +from django.http import HttpResponseNotAllowed from django.utils.module_loading import autodiscover_modules @@ -50,6 +51,9 @@ def autodiscover(): class ModelResource(tastypie.resources.ModelResource): + def post_detail(self, request, **kwargs): + return HttpResponseNotAllowed(["GET"]) + def generate_cache_key(self, *args, **kwargs): """ Creates a unique-enough cache key. diff --git a/ietf/api/management/commands/makeresources.py b/ietf/api/management/commands/makeresources.py index 889b2cdfb51..8e2c6f4b5cf 100644 --- a/ietf/api/management/commands/makeresources.py +++ b/ietf/api/management/commands/makeresources.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2014-2020, All Rights Reserved +# Copyright The IETF Trust 2014-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,7 +15,7 @@ from django.template import Template, Context from django.utils import timezone -from tastypie.resources import ModelResource +from ietf.api import ModelResource resource_head_template = """# Copyright The IETF Trust {{date}}, All Rights Reserved @@ -23,7 +23,7 @@ # Generated by the makeresources management command {{date}} -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 0e45ee3b391..ca605fe6441 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -805,7 +805,7 @@ class RfcFileSerializer(serializers.Serializer): # works. allowed_extensions = ( ".html", - ".json", + ".json", # deprecated - accepted but ignored, datatracker generates its own ".notprepped.xml", ".pdf", ".txt", diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 887969cec15..87f7d684d32 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -12,6 +12,7 @@ from importlib import import_module from pathlib import Path from random import randrange +from urllib.parse import urljoin from django.apps import apps from django.conf import settings @@ -1564,6 +1565,36 @@ def test_serializer_to_etree_handles_xml_invalid_control_chars(self): f"containing control character U+{ord(ch):04X}" ) + def test_post_detail_is_not_allowed(self): + """POST to a detail route returns 405 + + Added because default TastyPie behavior is a 500 due to a NotImplemented + exception. + """ + r = self.client.get("/api/v1", headers={"Accept": "application/json"}) + self.assertValidJSONResponse(r) + resource_list = r.json() + for name in self.apps: + r = self.client.get( + resource_list[name]["list_endpoint"], + headers={"Accept": "application/json"}, + ) + self.assertValidJSONResponse(r) + app_resources = r.json() + model_list = apps.get_app_config(name).get_models() + for model in model_list: + model_name = model._meta.model_name + assert model_name + detail_url = urljoin( + app_resources[model_name]["list_endpoint"], "some-id/" + ) + r = self.client.post(detail_url) + self.assertEqual( + r.status_code, + 405, + f"POST to {name}.{model_name} detail should return 405 status", + ) + class RfcdiffSupportTests(TestCase): diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 72e0d197857..5748a7caa60 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -224,6 +224,7 @@ def test_notify_rfc_published(self, mock_task_delay): self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.update_rfc_json_task") @mock.patch("ietf.api.views_rpc.rebuild_reference_relations_task") @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task") @mock.patch("ietf.api.views_rpc.trigger_red_precomputer_task") @@ -232,6 +233,7 @@ def test_upload_rfc_files( mock_trigger_red_task, mock_update_searchindex_task, mock_rebuild_relations, + mock_update_rfc_json, ): def _valid_post_data(): """Generate a valid post data dict @@ -292,7 +294,7 @@ def _valid_post_data(): ContentFile(b"", "myfile.txt"), ContentFile(b"", "myfile.html"), ContentFile(b"", "myfile.pdf"), - ContentFile(b"", "myfile.json"), + ContentFile(b"", "myfile.json"), # deprecated! ContentFile(b"", "myfile.notprepped.xml"), ] }, @@ -346,18 +348,19 @@ def _valid_post_data(): # valid post mock_trigger_red_task.delay.reset_mock() - r = self.client.post( - url, - _valid_post_data(), - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) + with self.captureOnCommitCallbacks(execute=True): + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) self.assertEqual(r.status_code, 200) self.assertEqual( mock_update_searchindex_task.delay.call_args, mock.call(rfc.rfc_number), ) - for extension in ["xml", "txt", "html", "pdf", "json"]: + for extension in ["xml", "txt", "html", "pdf"]: # no "json"! filename = f"{rfc.name}.{extension}" self.assertEqual( (rfc_path / filename).read_text(), @@ -389,6 +392,12 @@ def _valid_post_data(): b"This is .notprepped.xml", ".notprepped.xml blob should contain the expected content", ) + # special case for json (deprecated and should be ignored) + self.assertFalse((rfc_path / f"{rfc.name}.json").exists()) + self.assertFalse( + Blob.objects.filter(bucket="rfc", name=f"json/{rfc.name}.json").exists() + ) + # Confirm that the red precomputer was triggered correctly self.assertTrue(mock_trigger_red_task.delay.called) _, mock_kwargs = mock_trigger_red_task.delay.call_args @@ -404,6 +413,10 @@ def _valid_post_data(): _, mock_kwargs = mock_rebuild_relations.delay.call_args self.assertIn("doc_names", mock_kwargs) self.assertEqual(mock_kwargs["doc_names"], [rfc.name]) + # Confirm rfc JSON creation was triggered correctly + self.assertTrue(mock_update_rfc_json.delay.called) + mock_args, _ = mock_update_rfc_json.delay.call_args + self.assertEqual(mock_args[0], [rfc.rfc_number]) # re-post with replace = False should now fail mock_update_searchindex_task.reset_mock() diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 0c9e98e2dc0..21d59e3f14c 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2023-2026, All Rights Reserved import os import shutil +from functools import partial from pathlib import Path from tempfile import TemporaryDirectory @@ -463,7 +464,9 @@ def post(self, request): {d.rfc_number for d in rfc.related_that_doc(("updates", "obs"))} ) if related_numbers: - transaction.on_commit(lambda: update_rfc_json_task.delay(related_numbers)) + # Only update json for related rfcs. We can't generate json for _this_ rfc + # until we receive the files and know what formats we have. + transaction.on_commit(partial(update_rfc_json_task.delay, related_numbers)) return Response(NotificationAckSerializer().data) @@ -523,6 +526,10 @@ def post(self, request): for upfile in uploaded_files: uploaded_filename = Path(upfile.name) # name supplied by request uploaded_ext = "".join(uploaded_filename.suffixes) + # Ignore json files, which are deprecated. Remove this when purple no + # longer tries to send them. + if uploaded_ext == ".json": + continue tempfile_path = tmpfile_stem.with_suffix(uploaded_ext) with tempfile_path.open("wb") as dest: for chunk in upfile.chunks(): @@ -565,9 +572,10 @@ def post(self, request): trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating)) # Trigger search index update update_rfc_searchindex_task.delay(rfc.rfc_number) - # Trigger reference relation srebuild + # Trigger reference relations rebuild rebuild_reference_relations_task.delay(doc_names=[rfc.name]) - + # Build rfc json (json for related rfcs was updated in RfcPubNotificationView) + transaction.on_commit(partial(update_rfc_json_task.delay, [rfc.rfc_number])) return Response(NotificationAckSerializer().data) diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index 1a178c6f31d..0cf83e4525f 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -128,6 +128,13 @@ def states(obj, create, extracted, **kwargs): else: obj.set_state(State.objects.get(type_id='rfc',slug='published')) + @factory.post_generation + def authors(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument + # Override base class, creating RfcAuthor instead of DocumentAuthor + if create and extracted: + for person in extracted: + RfcAuthorFactory(document=obj, person=person) + class IndividualDraftFactory(BaseDocumentFactory): diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index ae5df641c29..e72cc04ff34 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -138,7 +138,7 @@ def prettystdname(string, space=" "): @register.filter def rfceditor_info_url(rfcnum : str): """Link to the RFC editor info page for an RFC""" - return urljoin(settings.RFC_EDITOR_INFO_BASE_URL, f'rfc{rfcnum}') + return urljoin(settings.RFC_EDITOR_INFO_BASE_URL, f'rfc{rfcnum}/') def doc_name(name): diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index ff4461d4666..86731000730 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -36,9 +36,9 @@ import debug # pyflakes:ignore -from ietf.doc.models import ( Document, DocRelationshipName, RelatedDocument, State, - DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent, BallotType, - EditedAuthorsDocEvent, StateType) +from ietf.doc.models import (Document, DocRelationshipName, RelatedDocument, State, + DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent, BallotType, + EditedAuthorsDocEvent, StateType, RfcAuthor) from ietf.doc.factories import (DocumentFactory, DocEventFactory, CharterFactory, ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, @@ -73,7 +73,7 @@ from ietf.utils.mail import get_payload_text, outbox, empty_outbox from ietf.utils.test_utils import login_testing_unauthorized, unicontent from ietf.utils.test_utils import TestCase -from ietf.utils.text import normalize_text +from ietf.utils.text import normalize_text, texescape from ietf.utils.timezone import date_today, datetime_today, DEADLINE_TZINFO, RPC_TZINFO from ietf.doc.utils_search import AD_WORKLOAD @@ -2133,9 +2133,11 @@ def test_document_bibtex(self): doc = factory() url = urlreverse("ietf.doc.views_doc.document_bibtex", kwargs=dict(name=doc.name)) r = self.client.get(url) - self.assertEqual(r.status_code, 404) + self.assertEqual(r.status_code, 404) + authors = PersonFactory.create_batch(2) rfc = WgRfcFactory.create( - time=datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo(settings.TIME_ZONE)) + time=datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo(settings.TIME_ZONE)), + authors=authors, ) num = rfc.rfc_number DocEventFactory.create( @@ -2152,7 +2154,12 @@ def test_document_bibtex(self): self.assertEqual(entry["doi"], "10.17487/RFC%s" % num) self.assertEqual(entry["year"], "2010") self.assertEqual(entry["month"].lower()[0:3], "oct") - self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}") + self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}/") + escaped_author_names = [ + texescape(ra.titlepage_name) + for ra in RfcAuthor.objects.filter(document=rfc) + ] + self.assertEqual(entry["author"], " and ".join(escaped_author_names)) # self.assertNotIn("day", entry) @@ -2188,7 +2195,7 @@ def test_document_bibtex(self): self.assertEqual(entry["year"], "1990") self.assertEqual(entry["month"].lower()[0:3], "apr") self.assertEqual(entry["day"], "1") - self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}") + self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}/") draft = IndividualDraftFactory.create() docname = "%s-%s" % (draft.name, draft.rev) diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index b406dd31528..ac86d7cf4b4 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -34,6 +34,7 @@ RfcAuthorFactory, WgDraftFactory, WgRfcFactory, ) +from ietf.doc.models import RfcAuthor from ietf.group.factories import GroupFactory, GroupHistoryFactory, RoleFactory, RoleHistoryFactory from ietf.group.models import Group, Role from ietf.meeting.factories import MeetingFactory, AttendedFactory, RegistrationFactory @@ -2535,19 +2536,6 @@ def test_get_qualified_author_queryset(self): people, # does not include extra_person! ) - # Now add an RfcAuthor for only one of the two authors to the RFC. This should - # remove the other author from the eligibility list because the DocumentAuthor - # records are no longer used. - RfcAuthorFactory( - document=rfc, - person=people[0], - titlepage_name="P. Zero", - ) - self.assertCountEqual( - get_qualified_author_queryset(base_qs, now - 5 * one_year, now), - [people[0]], - ) - class rfc8713EligibilityTests(TestCase): @@ -2875,7 +2863,9 @@ def test_elig_by_author(self): self.assertFalse(is_eligible(person,nomcom)) self.assertEqual(set(list_eligible(nomcom=nomcom)),set(eligible)) - Person.objects.filter(pk__in=[p.pk for p in eligible.union(ineligible)]).delete() + people_pks_to_delete = [p.pk for p in eligible.union(ineligible)] + RfcAuthor.objects.filter(person__pk__in=people_pks_to_delete).delete() + Person.objects.filter(pk__in=people_pks_to_delete).delete() class rfc9389EligibilityTests(TestCase): diff --git a/ietf/review/resources.py b/ietf/review/resources.py index f79d6dfc221..546ab50a5ca 100644 --- a/ietf/review/resources.py +++ b/ietf/review/resources.py @@ -1,9 +1,9 @@ -# Copyright The IETF Trust 2016-2019, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2016-06-14 04:21 PDT -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache diff --git a/ietf/static/css/document_html.scss b/ietf/static/css/document_html.scss index 47ef8d64b47..5a4282195f9 100644 --- a/ietf/static/css/document_html.scss +++ b/ietf/static/css/document_html.scss @@ -205,10 +205,6 @@ display: inline; } - .newpage { - margin-top: -1.25em; - } - } tbody.meta tr { diff --git a/ietf/stats/resources.py b/ietf/stats/resources.py index 59722c505ef..21e2c171acc 100644 --- a/ietf/stats/resources.py +++ b/ietf/stats/resources.py @@ -1,9 +1,9 @@ -# Copyright The IETF Trust 2017-2019, All Rights Reserved +# Copyright The IETF Trust 2017-2026, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2017-02-15 10:10 PST -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index ad361d31b26..abe23c1a643 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -40,7 +40,7 @@ from ietf.meeting.factories import MeetingFactory from ietf.name.models import DraftSubmissionStateName, FormalLanguageName from ietf.person.models import Person -from ietf.person.factories import UserFactory, PersonFactory +from ietf.person.factories import UserFactory, PersonFactory, EmailFactory from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactory from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm from ietf.submit.models import Submission, Preapproval, SubmissionExtResource @@ -1872,10 +1872,8 @@ def test_submit_bad_author_email(self): name = "draft-authorname-testing-bademail" rev = "00" - author = PersonFactory() - email = author.email_set.first() - email.address = '@bad.email' - email.save() + author = PersonFactory(default_emails=False) + EmailFactory(person=author, primary=True, address="@bad.email") status_url, _ = self.do_submission(name=name, rev=rev, author=author, formats=('xml',)) r = self.client.get(status_url) diff --git a/ietf/templates/doc/document_bibtex.bib b/ietf/templates/doc/document_bibtex.bib index 5e52ec3c58d..71b7498351a 100644 --- a/ietf/templates/doc/document_bibtex.bib +++ b/ietf/templates/doc/document_bibtex.bib @@ -6,8 +6,8 @@ {% if doc.type_id == "rfc" %} {% if doc.stream|slugify == "legacy" %} % Datatracker information for RFCs on the Legacy Stream is unfortunately often -% incorrect. Please correct the bibtex below based on the information in the -% actual RFC at https://rfc-editor.org/rfc/rfc{{ doc.rfc_number }}.txt +% incorrect. Please correct the bibtex below based on the information available +% at {{ doc.rfc_number|rfceditor_info_url }} {% endif %} @misc{% templatetag openbrace %}rfc{{ doc.rfc_number }}, series = {Request for Comments}, @@ -25,7 +25,7 @@ @techreport{ publisher = {% templatetag openbrace %}Internet Engineering Task Force{% templatetag closebrace %}, note = {% templatetag openbrace %}Work in Progress{% templatetag closebrace %}, url = {% templatetag openbrace %}{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name=doc.name rev=doc.rev %}{% templatetag closebrace %},{% endif %} - author = {% templatetag openbrace %}{% for author in doc.documentauthor_set.all %}{{ author.person.name|texescape}}{% if not forloop.last %} and {% endif %}{% endfor %}{% templatetag closebrace %}, + author = {% templatetag openbrace %}{% for author in doc.author_persons_or_names %}{% firstof author.titlepage_name|texescape author.person.name|texescape %}{% if not forloop.last %} and {% endif %}{% endfor %}{% templatetag closebrace %}, title = {% templatetag openbrace %}{% templatetag openbrace %}{{doc.title|texescape}}{% templatetag closebrace %}{% templatetag closebrace %}, pagetotal = {{ doc.pages }}, year = {{ doc.pub_date.year }}, diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index 9b967c013bf..57194144989 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -666,17 +666,14 @@ Nits - {% if user|has_role:"Area Director" %} - {# IDNITS3 is an experimental service, so only show it to Area Directors #} - Nits-v3 (Experimental) + Nits v3 - {% endif %} =3.2.0 # deprecated - need to replace with Django's JSONField jsonschema[format]>=4.25.0 jwcrypto>=1.5.6 # for signed notifications - this is aspirational, and is not really used. logging_tree>=1.10 # Used only by the showloggers management command -lxml>=6.0.0 +lxml>=6.1.0 markdown>=3.8.0 types-markdown>=3.8.0 mock>=5.2.0 # should replace with unittest.mock and remove dependency @@ -64,7 +64,7 @@ opentelemetry-exporter-otlp-proto-http>=1.38.0 pillow>=11.3.0 psycopg2>=2.9.10 pyang>=2.6.1 -pydyf>=0.11.0 +pydyf>=0.12.1 pyflakes>=3.4.0 pyopenssl>=25.1.0 # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency pyquery>=2.0.1 @@ -92,4 +92,4 @@ weasyprint>=66.0 xml2rfc>=3.30.0 xym>=0.6,<0.10.0 zxcvbn>=4.5.0 -types-zxcvbn~=4.5.0.20250223 # match zxcvbn version +types-zxcvbn~=4.5.0.20260518 # match zxcvbn version