ci: merge main into feat/clarity

.devcontainer/devcontainer.json

@@ -33,23 +33,16 @@
                 "oderwat.indent-rainbow",
                 "redhat.vscode-yaml",
                 "spmeesseman.vscode-taskexplorer",
-                "visualstudioexptteam.vscodeintellicode"
+                "visualstudioexptteam.vscodeintellicode",
+                "ms-python.pylint"
             ],
             "settings": {
                 "terminal.integrated.defaultProfile.linux": "zsh",
                 "python.pythonPath": "/usr/local/bin/python",
                 "python.languageServer": "Default",
-                "python.linting.enabled": true,
-                "python.linting.pylintEnabled": true,
                 "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
                 "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
                 "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
-                "python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
-                "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
-                "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
-                "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
-                "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
-                "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
                 "python.testing.pytestArgs": [
                     "ietf"
                 ],

dev/deploy-to-container/cli.js

@@ -245,7 +245,7 @@ async function main () {
     name: `dt-app-${branch}`,
     Hostname: `dt-app-${branch}`,
     Env: [
-      `LETSENCRYPT_HOST=${hostname}`,
+      // `LETSENCRYPT_HOST=${hostname}`,
       `VIRTUAL_HOST=${hostname}`,
       `VIRTUAL_PORT=8000`,
       `PGHOST=dt-db-${branch}`

ietf/api/views.py

@@ -210,11 +210,14 @@ def err(code, text):
                 except (NomCom.DoesNotExist, NomCom.MultipleObjectsReturned):
                     nomcom = None
                 if nomcom:
-                    Volunteer.objects.create(
+                    Volunteer.objects.get_or_create(
                         nomcom=nomcom,
                         person=object.person,
-                        affiliation=data['affiliation'],
-                        origin='registration')
+                        defaults={
+                            "affiliation": data["affiliation"],
+                            "origin": "registration"
+                        }
+                    )
             return HttpResponse(response, status=202, content_type='text/plain')
     else:
         return HttpResponse(status=405)

ietf/doc/models.py

@@ -649,10 +649,10 @@ def referenced_by(self):
                 source__states__slug="active",
             )
             | models.Q(source__type__slug="rfc")
-        )
-
+        ).distinct()
 
     def referenced_by_rfcs(self):
+        """Get refs to this doc from RFCs"""
         return self.relations_that(("refnorm", "refinfo", "refunk", "refold")).filter(
             source__type__slug="rfc"
         )
@@ -675,6 +675,13 @@ def contains(self):
     def part_of(self):
         return self.related_that("contains")
 
+    def referenced_by_rfcs_as_rfc_or_draft(self):
+        """Get refs to this doc, or a draft/rfc it came from, from an RFC"""
+        refs_to = self.referenced_by_rfcs()
+        if self.type_id == "rfc" and self.came_from_draft():
+            refs_to |= self.came_from_draft().referenced_by_rfcs()
+        return refs_to
+
     class Meta:
         abstract = True
 

ietf/doc/tests.py

@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2012-2020, All Rights Reserved
+# Copyright The IETF Trust 2012-2023, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -40,7 +40,7 @@
     ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, 
     IndividualRfcFactory, StateDocEventFactory, BallotPositionDocEventFactory, 
     BallotDocEventFactory, DocumentAuthorFactory, NewRevisionDocEventFactory,
-    StatusChangeFactory, DocExtResourceFactory, RgDraftFactory)
+    StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory)
 from ietf.doc.forms import NotifyForm
 from ietf.doc.fields import SearchableDocumentsField
 from ietf.doc.utils import create_ballot_if_not_open, uppercase_std_abbreviated_name
@@ -156,6 +156,23 @@ def test_search(self):
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
 
+    def test_search_became_rfc(self):
+        draft = WgDraftFactory()
+        rfc = WgRfcFactory()
+        draft.set_state(State.objects.get(type="draft", slug="rfc"))
+        draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc)
+        base_url = urlreverse('ietf.doc.views_search.search')
+
+        # find by RFC
+        r = self.client.get(base_url + f"?rfcs=on&name={rfc.name}")
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, rfc.title)
+
+        # find by draft
+        r = self.client.get(base_url + f"?activedrafts=on&rfcs=on&name={draft.name}")
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, rfc.title)
+
     def test_search_for_name(self):
         draft = WgDraftFactory(name='draft-ietf-mars-test',group=GroupFactory(acronym='mars',parent=Group.objects.get(acronym='farfut')),authors=[PersonFactory()],ad=PersonFactory())
         draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="pub-req"))
@@ -1948,6 +1965,12 @@ def _parse_bibtex_response(self, response) -> dict:
 
     @override_settings(RFC_EDITOR_INFO_BASE_URL='https://www.rfc-editor.ietf.org/info/')
     def test_document_bibtex(self):
+
+        for factory in [CharterFactory, BcpFactory, StatusChangeFactory, ConflictReviewFactory]: # Should be extended to all other doc types
+            doc = factory()
+            url = urlreverse("ietf.doc.views_doc.document_bibtex", kwargs=dict(name=doc.name))
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 404)          
         rfc = WgRfcFactory.create(
             time=datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo(settings.TIME_ZONE))
         )
@@ -2951,4 +2974,51 @@ def test_revisions(self):
         self.assertEqual(draft.revisions_by_dochistory(),[f"{i:02d}" for i in range(8,10)])
         self.assertEqual(draft.revisions_by_newrevisionevent(),[f"{i:02d}" for i in [*range(0,5), *range(6,10)]])      
 
+    def test_referenced_by_rfcs(self):
+        # n.b., no significance to the ref* values in this test
+        referring_draft = WgDraftFactory()
+        (rfc, referring_rfc) = WgRfcFactory.create_batch(2)
+        rfc.targets_related.create(relationship_id="refnorm", source=referring_draft)
+        rfc.targets_related.create(relationship_id="refnorm", source=referring_rfc)
+        self.assertCountEqual(
+            rfc.referenced_by_rfcs(),
+            rfc.targets_related.filter(source=referring_rfc),
+        )
+
+    def test_referenced_by_rfcs_as_rfc_or_draft(self):
+        # n.b., no significance to the ref* values in this test
+        draft = WgDraftFactory()
+        rfc = WgRfcFactory()
+        draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc)
+
+        # Draft referring to the rfc and the draft - should not be reported at all
+        draft_referring_to_both = WgDraftFactory()
+        draft_referring_to_both.relateddocument_set.create(relationship_id="refnorm", target=draft)
+        draft_referring_to_both.relateddocument_set.create(relationship_id="refnorm", target=rfc)
+
+        # RFC referring only to the draft - should be reported for either the draft or the rfc
+        rfc_referring_to_draft = WgRfcFactory()
+        rfc_referring_to_draft.relateddocument_set.create(relationship_id="refinfo", target=draft)
+
+        # RFC referring only to the rfc - should be reported only for the rfc
+        rfc_referring_to_rfc = WgRfcFactory()
+        rfc_referring_to_rfc.relateddocument_set.create(relationship_id="refinfo", target=rfc)
+
+        # RFC referring only to the rfc - should be reported only for the rfc
+        rfc_referring_to_rfc = WgRfcFactory()
+        rfc_referring_to_rfc.relateddocument_set.create(relationship_id="refinfo", target=rfc)
+
+        # RFC referring to the rfc and the draft - should be reported for both
+        rfc_referring_to_both = WgRfcFactory()
+        rfc_referring_to_both.relateddocument_set.create(relationship_id="refnorm", target=draft)
+        rfc_referring_to_both.relateddocument_set.create(relationship_id="refnorm", target=rfc)
+
+        self.assertCountEqual(
+            draft.referenced_by_rfcs_as_rfc_or_draft(),
+            draft.targets_related.filter(source__type="rfc"),
+        )
 
+        self.assertCountEqual(
+            rfc.referenced_by_rfcs_as_rfc_or_draft(),
+            draft.targets_related.filter(source__type="rfc") | rfc.targets_related.filter(source__type="rfc"),
+        )

ietf/doc/views_doc.py

@@ -1262,6 +1262,9 @@ def document_bibtex(request, name, rev=None):
 
     doc = get_object_or_404(Document, name=name)
 
+    if doc.type_id not in ["rfc", "draft"]:
+        raise Http404()
+
     doi = None
     draft_became_rfc = None
     replaced_by = None
@@ -1437,8 +1440,26 @@ def document_references(request, name):
     return render(request, "doc/document_references.html",dict(doc=doc,refs=sorted(refs,key=lambda x:x.target.name),))
 
 def document_referenced_by(request, name):
+    """View documents that reference the named document
+
+    The view lists both direct references to a the named document, plus references to
+    related other documents. For a draft that became an RFC, this will include references
+    to the RFC. For an RFC, this will include references to the draft it came from, if any.
+    For a subseries document, this will include references to any of the RFC documents it
+    contains. 
+
+    In the rendered output, a badge is applied to indicate the name of the document the
+    reference actually targeted. E.g., on the display for a draft that became RFC NNN,
+    references included because they point to that RFC would be shown with a tag "As RFC NNN".
+    The intention is to make the "Referenced By" page useful for finding related work while
+    accurately reflecting the actual reference relationships.     
+    """
     doc = get_object_or_404(Document,name=name)
     refs = doc.referenced_by()
+    if doc.came_from_draft():
+        refs |= doc.came_from_draft().referenced_by()
+    if doc.became_rfc():
+        refs |= doc.became_rfc().referenced_by()
     if doc.type_id in ["bcp","std","fyi"]:
         for rfc in doc.contains():
             refs |= rfc.referenced_by()
@@ -2167,22 +2188,48 @@ def idnits2_state(request, name, rev=None):
     if doc.type_id == "rfc":
         draft = doc.came_from_draft()
         if draft:
-            zero_revision = NewRevisionDocEvent.objects.filter(doc=draft,rev='00').first()
+            zero_revision = NewRevisionDocEvent.objects.filter(
+                doc=draft, rev="00"
+            ).first()
     else:
-        zero_revision = NewRevisionDocEvent.objects.filter(doc=doc,rev='00').first()
+        zero_revision = NewRevisionDocEvent.objects.filter(doc=doc, rev="00").first()
     if zero_revision:
         doc.created = zero_revision.time
     else:
-        doc.created = doc.docevent_set.order_by('-time').first().time
+        if doc.type_id == "draft":
+            if doc.became_rfc():
+                interesting_event = (
+                    doc.became_rfc()
+                    .docevent_set.filter(type="published_rfc")
+                    .order_by("-time")
+                    .first()
+                )
+            else:
+                interesting_event = doc.docevent_set.order_by(
+                    "-time"
+                ).first()  # Is taking the most _recent_ instead of the oldest event correct?
+        else:  # doc.type_id == "rfc"
+            interesting_event = (
+                doc.docevent_set.filter(type="published_rfc").order_by("-time").first()
+            )
+        doc.created = interesting_event.time
     if doc.std_level:
         doc.deststatus = doc.std_level.name
     elif doc.intended_std_level:
         doc.deststatus = doc.intended_std_level.name
     else:
         text = doc.text()
         if text:
-            parsed_draft = PlaintextDraft(text=doc.text(), source=name, name_from_source=False)
+            parsed_draft = PlaintextDraft(
+                text=doc.text(), source=name, name_from_source=False
+            )
             doc.deststatus = parsed_draft.get_status()
         else:
-            doc.deststatus="Unknown"
-    return render(request, 'doc/idnits2-state.txt', context={'doc':doc}, content_type='text/plain;charset=utf-8')    
+            doc.deststatus = "Unknown"
+    return render(
+        request,
+        "doc/idnits2-state.txt",
+        context={"doc": doc},
+        content_type="text/plain;charset=utf-8",
+    )
+

ietf/doc/views_search.py

@@ -211,6 +211,9 @@ def retrieve_search_results(form, all_types=False):
                     Q(targets_related__source__title__icontains=singlespace, targets_related__relationship_id="contains"),
                 ])
 
+        if query["rfcs"]:
+            queries.extend([Q(targets_related__source__name__icontains=look_for, targets_related__relationship_id="became_rfc")])
+
         combined_query = reduce(operator.or_, queries)
         docs = docs.filter(combined_query).distinct()
 

ietf/group/views.py

@@ -1296,7 +1296,7 @@ def stream_documents(request, acronym):
     qs = Document.objects.filter(stream=acronym).filter(
         Q(type_id="draft", states__type="draft", states__slug="active")
         | Q(type_id="rfc")
-    )
+    ).distinct()
     docs, meta = prepare_document_table(request, qs, max_results=1000)
     return render(request, 'group/stream_documents.html', {'stream':stream, 'docs':docs, 'meta':meta, 'editable':editable } )