diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 14a0d5ea90..ac7854f265 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -33,23 +33,16 @@ "oderwat.indent-rainbow", "redhat.vscode-yaml", "spmeesseman.vscode-taskexplorer", - "visualstudioexptteam.vscodeintellicode" + "visualstudioexptteam.vscodeintellicode", + "ms-python.pylint" ], "settings": { "terminal.integrated.defaultProfile.linux": "zsh", "python.pythonPath": "/usr/local/bin/python", "python.languageServer": "Default", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", "python.formatting.blackPath": "/usr/local/py-utils/bin/black", "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", - "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", - "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", - "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", - "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", - "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", - "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", "python.testing.pytestArgs": [ "ietf" ], diff --git a/dev/deploy-to-container/cli.js b/dev/deploy-to-container/cli.js index 1c3d466286..a22c746ae2 100644 --- a/dev/deploy-to-container/cli.js +++ b/dev/deploy-to-container/cli.js @@ -245,7 +245,7 @@ async function main () { name: `dt-app-${branch}`, Hostname: `dt-app-${branch}`, Env: [ - `LETSENCRYPT_HOST=${hostname}`, + // `LETSENCRYPT_HOST=${hostname}`, `VIRTUAL_HOST=${hostname}`, `VIRTUAL_PORT=8000`, `PGHOST=dt-db-${branch}` diff --git a/ietf/api/views.py b/ietf/api/views.py index e587b37121..73b873f5f3 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -210,11 +210,14 @@ def err(code, text): except (NomCom.DoesNotExist, NomCom.MultipleObjectsReturned): nomcom = None if nomcom: - Volunteer.objects.create( + Volunteer.objects.get_or_create( nomcom=nomcom, person=object.person, - affiliation=data['affiliation'], - origin='registration') + defaults={ + "affiliation": data["affiliation"], + "origin": "registration" + } + ) return HttpResponse(response, status=202, content_type='text/plain') else: return HttpResponse(status=405) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index fdda23ecd1..434096929a 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -649,10 +649,10 @@ def referenced_by(self): source__states__slug="active", ) | models.Q(source__type__slug="rfc") - ) - + ).distinct() def referenced_by_rfcs(self): + """Get refs to this doc from RFCs""" return self.relations_that(("refnorm", "refinfo", "refunk", "refold")).filter( source__type__slug="rfc" ) @@ -675,6 +675,13 @@ def contains(self): def part_of(self): return self.related_that("contains") + def referenced_by_rfcs_as_rfc_or_draft(self): + """Get refs to this doc, or a draft/rfc it came from, from an RFC""" + refs_to = self.referenced_by_rfcs() + if self.type_id == "rfc" and self.came_from_draft(): + refs_to |= self.came_from_draft().referenced_by_rfcs() + return refs_to + class Meta: abstract = True diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index f14b5b1af0..71ec23906c 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2012-2020, All Rights Reserved +# Copyright The IETF Trust 2012-2023, All Rights Reserved # -*- coding: utf-8 -*- @@ -40,7 +40,7 @@ ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, IndividualRfcFactory, StateDocEventFactory, BallotPositionDocEventFactory, BallotDocEventFactory, DocumentAuthorFactory, NewRevisionDocEventFactory, - StatusChangeFactory, DocExtResourceFactory, RgDraftFactory) + StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory) from ietf.doc.forms import NotifyForm from ietf.doc.fields import SearchableDocumentsField from ietf.doc.utils import create_ballot_if_not_open, uppercase_std_abbreviated_name @@ -156,6 +156,23 @@ def test_search(self): self.assertEqual(r.status_code, 200) self.assertContains(r, draft.title) + def test_search_became_rfc(self): + draft = WgDraftFactory() + rfc = WgRfcFactory() + draft.set_state(State.objects.get(type="draft", slug="rfc")) + draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc) + base_url = urlreverse('ietf.doc.views_search.search') + + # find by RFC + r = self.client.get(base_url + f"?rfcs=on&name={rfc.name}") + self.assertEqual(r.status_code, 200) + self.assertContains(r, rfc.title) + + # find by draft + r = self.client.get(base_url + f"?activedrafts=on&rfcs=on&name={draft.name}") + self.assertEqual(r.status_code, 200) + self.assertContains(r, rfc.title) + def test_search_for_name(self): draft = WgDraftFactory(name='draft-ietf-mars-test',group=GroupFactory(acronym='mars',parent=Group.objects.get(acronym='farfut')),authors=[PersonFactory()],ad=PersonFactory()) draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="pub-req")) @@ -1948,6 +1965,12 @@ def _parse_bibtex_response(self, response) -> dict: @override_settings(RFC_EDITOR_INFO_BASE_URL='https://www.rfc-editor.ietf.org/info/') def test_document_bibtex(self): + + for factory in [CharterFactory, BcpFactory, StatusChangeFactory, ConflictReviewFactory]: # Should be extended to all other doc types + doc = factory() + url = urlreverse("ietf.doc.views_doc.document_bibtex", kwargs=dict(name=doc.name)) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) rfc = WgRfcFactory.create( time=datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo(settings.TIME_ZONE)) ) @@ -2951,4 +2974,51 @@ def test_revisions(self): self.assertEqual(draft.revisions_by_dochistory(),[f"{i:02d}" for i in range(8,10)]) self.assertEqual(draft.revisions_by_newrevisionevent(),[f"{i:02d}" for i in [*range(0,5), *range(6,10)]]) + def test_referenced_by_rfcs(self): + # n.b., no significance to the ref* values in this test + referring_draft = WgDraftFactory() + (rfc, referring_rfc) = WgRfcFactory.create_batch(2) + rfc.targets_related.create(relationship_id="refnorm", source=referring_draft) + rfc.targets_related.create(relationship_id="refnorm", source=referring_rfc) + self.assertCountEqual( + rfc.referenced_by_rfcs(), + rfc.targets_related.filter(source=referring_rfc), + ) + + def test_referenced_by_rfcs_as_rfc_or_draft(self): + # n.b., no significance to the ref* values in this test + draft = WgDraftFactory() + rfc = WgRfcFactory() + draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc) + + # Draft referring to the rfc and the draft - should not be reported at all + draft_referring_to_both = WgDraftFactory() + draft_referring_to_both.relateddocument_set.create(relationship_id="refnorm", target=draft) + draft_referring_to_both.relateddocument_set.create(relationship_id="refnorm", target=rfc) + + # RFC referring only to the draft - should be reported for either the draft or the rfc + rfc_referring_to_draft = WgRfcFactory() + rfc_referring_to_draft.relateddocument_set.create(relationship_id="refinfo", target=draft) + + # RFC referring only to the rfc - should be reported only for the rfc + rfc_referring_to_rfc = WgRfcFactory() + rfc_referring_to_rfc.relateddocument_set.create(relationship_id="refinfo", target=rfc) + + # RFC referring only to the rfc - should be reported only for the rfc + rfc_referring_to_rfc = WgRfcFactory() + rfc_referring_to_rfc.relateddocument_set.create(relationship_id="refinfo", target=rfc) + + # RFC referring to the rfc and the draft - should be reported for both + rfc_referring_to_both = WgRfcFactory() + rfc_referring_to_both.relateddocument_set.create(relationship_id="refnorm", target=draft) + rfc_referring_to_both.relateddocument_set.create(relationship_id="refnorm", target=rfc) + + self.assertCountEqual( + draft.referenced_by_rfcs_as_rfc_or_draft(), + draft.targets_related.filter(source__type="rfc"), + ) + self.assertCountEqual( + rfc.referenced_by_rfcs_as_rfc_or_draft(), + draft.targets_related.filter(source__type="rfc") | rfc.targets_related.filter(source__type="rfc"), + ) diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 8408f5a0cd..e0bf328e89 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -1262,6 +1262,9 @@ def document_bibtex(request, name, rev=None): doc = get_object_or_404(Document, name=name) + if doc.type_id not in ["rfc", "draft"]: + raise Http404() + doi = None draft_became_rfc = None replaced_by = None @@ -1437,8 +1440,26 @@ def document_references(request, name): return render(request, "doc/document_references.html",dict(doc=doc,refs=sorted(refs,key=lambda x:x.target.name),)) def document_referenced_by(request, name): + """View documents that reference the named document + + The view lists both direct references to a the named document, plus references to + related other documents. For a draft that became an RFC, this will include references + to the RFC. For an RFC, this will include references to the draft it came from, if any. + For a subseries document, this will include references to any of the RFC documents it + contains. + + In the rendered output, a badge is applied to indicate the name of the document the + reference actually targeted. E.g., on the display for a draft that became RFC NNN, + references included because they point to that RFC would be shown with a tag "As RFC NNN". + The intention is to make the "Referenced By" page useful for finding related work while + accurately reflecting the actual reference relationships. + """ doc = get_object_or_404(Document,name=name) refs = doc.referenced_by() + if doc.came_from_draft(): + refs |= doc.came_from_draft().referenced_by() + if doc.became_rfc(): + refs |= doc.became_rfc().referenced_by() if doc.type_id in ["bcp","std","fyi"]: for rfc in doc.contains(): refs |= rfc.referenced_by() @@ -2167,13 +2188,31 @@ def idnits2_state(request, name, rev=None): if doc.type_id == "rfc": draft = doc.came_from_draft() if draft: - zero_revision = NewRevisionDocEvent.objects.filter(doc=draft,rev='00').first() + zero_revision = NewRevisionDocEvent.objects.filter( + doc=draft, rev="00" + ).first() else: - zero_revision = NewRevisionDocEvent.objects.filter(doc=doc,rev='00').first() + zero_revision = NewRevisionDocEvent.objects.filter(doc=doc, rev="00").first() if zero_revision: doc.created = zero_revision.time else: - doc.created = doc.docevent_set.order_by('-time').first().time + if doc.type_id == "draft": + if doc.became_rfc(): + interesting_event = ( + doc.became_rfc() + .docevent_set.filter(type="published_rfc") + .order_by("-time") + .first() + ) + else: + interesting_event = doc.docevent_set.order_by( + "-time" + ).first() # Is taking the most _recent_ instead of the oldest event correct? + else: # doc.type_id == "rfc" + interesting_event = ( + doc.docevent_set.filter(type="published_rfc").order_by("-time").first() + ) + doc.created = interesting_event.time if doc.std_level: doc.deststatus = doc.std_level.name elif doc.intended_std_level: @@ -2181,8 +2220,16 @@ def idnits2_state(request, name, rev=None): else: text = doc.text() if text: - parsed_draft = PlaintextDraft(text=doc.text(), source=name, name_from_source=False) + parsed_draft = PlaintextDraft( + text=doc.text(), source=name, name_from_source=False + ) doc.deststatus = parsed_draft.get_status() else: - doc.deststatus="Unknown" - return render(request, 'doc/idnits2-state.txt', context={'doc':doc}, content_type='text/plain;charset=utf-8') + doc.deststatus = "Unknown" + return render( + request, + "doc/idnits2-state.txt", + context={"doc": doc}, + content_type="text/plain;charset=utf-8", + ) + diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 2e4231c5ac..789f32e387 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -211,6 +211,9 @@ def retrieve_search_results(form, all_types=False): Q(targets_related__source__title__icontains=singlespace, targets_related__relationship_id="contains"), ]) + if query["rfcs"]: + queries.extend([Q(targets_related__source__name__icontains=look_for, targets_related__relationship_id="became_rfc")]) + combined_query = reduce(operator.or_, queries) docs = docs.filter(combined_query).distinct() diff --git a/ietf/group/views.py b/ietf/group/views.py index 991a1b8d87..698963678a 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -1296,7 +1296,7 @@ def stream_documents(request, acronym): qs = Document.objects.filter(stream=acronym).filter( Q(type_id="draft", states__type="draft", states__slug="active") | Q(type_id="rfc") - ) + ).distinct() docs, meta = prepare_document_table(request, qs, max_results=1000) return render(request, 'group/stream_documents.html', {'stream':stream, 'docs':docs, 'meta':meta, 'editable':editable } ) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 47e2334f47..a57fcf63c1 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2009-2020, All Rights Reserved +# Copyright The IETF Trust 2009-2023, All Rights Reserved # -*- coding: utf-8 -*- import datetime import io @@ -6106,21 +6106,21 @@ def test_upload_minutes_agenda(self): test_file = BytesIO(b'this is some text for a test') test_file.name = "not_really.json" - r = self.client.post(url,dict(file=test_file)) + r = self.client.post(url,dict(submission_method="upload",file=test_file)) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('form .is-invalid')) test_file = BytesIO(b'this is some text for a test'*1510000) test_file.name = "not_really.pdf" - r = self.client.post(url,dict(file=test_file)) + r = self.client.post(url,dict(submission_method="upload",file=test_file)) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('form .is-invalid')) test_file = BytesIO(b'
') test_file.name = "not_really.html" - r = self.client.post(url,dict(file=test_file)) + r = self.client.post(url,dict(submission_method="upload",file=test_file)) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('form .is-invalid')) @@ -6128,7 +6128,7 @@ def test_upload_minutes_agenda(self): # Test html sanitization test_file = BytesIO(b'