ci: merge main to release

.devcontainer/devcontainer.json

@@ -33,23 +33,16 @@
                 "oderwat.indent-rainbow",
                 "redhat.vscode-yaml",
                 "spmeesseman.vscode-taskexplorer",
-                "visualstudioexptteam.vscodeintellicode"
+                "visualstudioexptteam.vscodeintellicode",
+                "ms-python.pylint"
             ],
             "settings": {
                 "terminal.integrated.defaultProfile.linux": "zsh",
                 "python.pythonPath": "/usr/local/bin/python",
                 "python.languageServer": "Default",
-                "python.linting.enabled": true,
-                "python.linting.pylintEnabled": true,
                 "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
                 "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
                 "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
-                "python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
-                "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
-                "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
-                "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
-                "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
-                "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
                 "python.testing.pytestArgs": [
                     "ietf"
                 ],

dev/deploy-to-container/cli.js

@@ -245,7 +245,7 @@ async function main () {
     name: `dt-app-${branch}`,
     Hostname: `dt-app-${branch}`,
     Env: [
-      `LETSENCRYPT_HOST=${hostname}`,
+      // `LETSENCRYPT_HOST=${hostname}`,
       `VIRTUAL_HOST=${hostname}`,
       `VIRTUAL_PORT=8000`,
       `PGHOST=dt-db-${branch}`

dev/deploy-to-container/start.sh

@@ -38,8 +38,5 @@ echo "Running Datatracker checks..."
 echo "Running Datatracker migrations..."
 /usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local
 
-echo "Syncing with the rfc-index"
-./ietf/bin/rfc-editor-index-updates -d 1969-01-01
-
 echo "Starting Datatracker..."
 ./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local

ietf/doc/models.py

@@ -634,6 +634,9 @@ def pdfized(self):
                 )
             except AssertionError:
                 pdf = None
+            except Exception as e:
+                log.log('weasyprint failed:'+str(e))
+                raise
             if pdf:
                 cache.set(cache_key, pdf, settings.PDFIZER_CACHE_TIME)
         return pdf
@@ -649,7 +652,7 @@ def referenced_by(self):
                 source__states__slug="active",
             )
             | models.Q(source__type__slug="rfc")
-        )
+        ).distinct()
 
     def referenced_by_rfcs(self):
         """Get refs to this doc from RFCs"""

ietf/doc/tests.py

@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2012-2020, All Rights Reserved
+# Copyright The IETF Trust 2012-2023, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -31,6 +31,8 @@
 
 from tastypie.test import ResourceTestCaseMixin
 
+from weasyprint.urls import URLFetchingError
+
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import ( Document, DocRelationshipName, RelatedDocument, State,
@@ -40,7 +42,7 @@
     ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, 
     IndividualRfcFactory, StateDocEventFactory, BallotPositionDocEventFactory, 
     BallotDocEventFactory, DocumentAuthorFactory, NewRevisionDocEventFactory,
-    StatusChangeFactory, DocExtResourceFactory, RgDraftFactory)
+    StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory)
 from ietf.doc.forms import NotifyForm
 from ietf.doc.fields import SearchableDocumentsField
 from ietf.doc.utils import create_ballot_if_not_open, uppercase_std_abbreviated_name
@@ -156,6 +158,23 @@ def test_search(self):
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
 
+    def test_search_became_rfc(self):
+        draft = WgDraftFactory()
+        rfc = WgRfcFactory()
+        draft.set_state(State.objects.get(type="draft", slug="rfc"))
+        draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc)
+        base_url = urlreverse('ietf.doc.views_search.search')
+
+        # find by RFC
+        r = self.client.get(base_url + f"?rfcs=on&name={rfc.name}")
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, rfc.title)
+
+        # find by draft
+        r = self.client.get(base_url + f"?activedrafts=on&rfcs=on&name={draft.name}")
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, rfc.title)
+
     def test_search_for_name(self):
         draft = WgDraftFactory(name='draft-ietf-mars-test',group=GroupFactory(acronym='mars',parent=Group.objects.get(acronym='farfut')),authors=[PersonFactory()],ad=PersonFactory())
         draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="pub-req"))
@@ -1948,6 +1967,12 @@ def _parse_bibtex_response(self, response) -> dict:
 
     @override_settings(RFC_EDITOR_INFO_BASE_URL='https://www.rfc-editor.ietf.org/info/')
     def test_document_bibtex(self):
+
+        for factory in [CharterFactory, BcpFactory, StatusChangeFactory, ConflictReviewFactory]: # Should be extended to all other doc types
+            doc = factory()
+            url = urlreverse("ietf.doc.views_doc.document_bibtex", kwargs=dict(name=doc.name))
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 404)          
         rfc = WgRfcFactory.create(
             time=datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo(settings.TIME_ZONE))
         )
@@ -2844,6 +2869,12 @@ def test_pdfized(self):
                 self.should_succeed(dict(name=draft.name,rev=f'{r:02d}',ext=ext))
         self.should_404(dict(name=draft.name,rev='02'))
 
+        with mock.patch('ietf.doc.models.DocumentInfo.pdfized', side_effect=URLFetchingError):
+            url = urlreverse(self.view, kwargs=dict(name=rfc.name))
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            self.assertContains(r, "Error while rendering PDF")
+
 class NotifyValidationTests(TestCase):
     def test_notify_validation(self):
         valid_values = [

ietf/doc/tests_draft.py

@@ -1483,6 +1483,42 @@ def test_confirm_submission(self):
         self.assertTrue("aread@" in outbox[-1]['To'])
         self.assertTrue("iesg-secretary@" in outbox[-1]['Cc'])
 
+    def test_confirm_submission_no_doc_ad(self):
+        url = urlreverse('ietf.doc.views_draft.to_iesg', kwargs=dict(name=self.docname))
+        self.client.login(username="marschairman", password="marschairman+password")
+
+        doc = Document.objects.get(name=self.docname)
+        RoleFactory(name_id='ad', group=doc.group, person=doc.ad)
+        e = DocEvent(type="changed_document", by=doc.ad, doc=doc, rev=doc.rev, desc="Remove doc AD")
+        e.save()
+        doc.ad = None
+        doc.save_with_history([e])
+
+        docevents_pre = set(doc.docevent_set.all())
+        mailbox_before = len(outbox)
+
+        r = self.client.post(url, dict(confirm="1"))
+        self.assertEqual(r.status_code, 302)
+
+        doc = Document.objects.get(name=self.docname)
+        self.assertTrue(doc.get_state('draft-iesg').slug=='pub-req')
+        self.assertTrue(doc.get_state('draft-stream-ietf').slug=='sub-pub')
+
+        self.assertCountEqual(doc.action_holders.all(), [doc.ad])
+
+        new_docevents = set(doc.docevent_set.all()) - docevents_pre
+        self.assertEqual(len(new_docevents), 5)
+        new_docevent_type_count = Counter([e.type for e in new_docevents])
+        self.assertEqual(new_docevent_type_count['changed_state'],2)
+        self.assertEqual(new_docevent_type_count['started_iesg_process'],1)
+        self.assertEqual(new_docevent_type_count['changed_action_holders'], 1)
+        self.assertEqual(new_docevent_type_count['changed_document'], 1)
+
+        self.assertEqual(len(outbox), mailbox_before + 1)
+        self.assertTrue("Publication has been requested" in outbox[-1]['Subject'])
+        self.assertTrue("aread@" in outbox[-1]['To'])
+        self.assertTrue("iesg-secretary@" in outbox[-1]['Cc'])
+
 
 
 class RequestPublicationTests(TestCase):

ietf/doc/tests_review.py

@@ -380,6 +380,25 @@ def test_assign_reviewer_after_reject(self):
         reviewer_label = q("option[value=\"{}\"]".format(reviewer_email.address)).text().lower()
         self.assertIn("rejected review of document before", reviewer_label)
 
+    def test_assign_reviewer_after_withdraw(self):
+        doc = WgDraftFactory()
+        review_team = ReviewTeamFactory()
+        rev_role = RoleFactory(group=review_team,person__user__username='reviewer',person__user__email='reviewer@example.com',name_id='reviewer')
+        RoleFactory(group=review_team,person__user__username='reviewsecretary',name_id='secr')
+        review_req = ReviewRequestFactory(team=review_team,doc=doc)
+        reviewer = rev_role.person.email_set.first()
+        ReviewAssignmentFactory(review_request=review_req, state_id='withdrawn', reviewer=reviewer)
+        req_url = urlreverse('ietf.doc.views_review.review_request', kwargs={ "name": doc.name, "request_id": review_req.pk })
+        assign_url = urlreverse('ietf.doc.views_review.assign_reviewer', kwargs={ "name": doc.name, "request_id": review_req.pk })
+
+        login_testing_unauthorized(self, "reviewsecretary", assign_url)
+        r = self.client.post(assign_url, { "action": "assign", "reviewer": reviewer.pk })
+        self.assertRedirects(r, req_url)
+        review_req = reload_db_objects(review_req)
+        assignment = review_req.reviewassignment_set.last()
+        self.assertEqual(assignment.state, ReviewAssignmentStateName.objects.get(slug='assigned'))
+        self.assertEqual(review_req.state, ReviewRequestStateName.objects.get(slug='assigned'))
+
     def test_previously_reviewed_replaced_doc(self):
         review_team = ReviewTeamFactory(acronym="reviewteam", name="Review Team", type_id="review", list_email="reviewteam@ietf.org", parent=Group.objects.get(acronym="farfut"))
         rev_role = RoleFactory(group=review_team,person__user__username='reviewer',person__user__email='reviewer@example.com',person__name='Some Reviewer',name_id='reviewer')

ietf/doc/views_doc.py

@@ -51,7 +51,6 @@
 from django import forms
 from django.contrib.staticfiles import finders
 
-
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import ( Document, DocHistory, DocEvent, BallotDocEvent, BallotType,
@@ -1064,7 +1063,10 @@ def document_pdfized(request, name, rev=None, ext=None):
     if not os.path.exists(doc.get_file_name()):
         raise Http404("File not found: %s" % doc.get_file_name())
 
-    pdf = doc.pdfized()
+    try:
+        pdf = doc.pdfized()
+    except Exception:
+        return render(request, "doc/weasyprint_failed.html")
     if pdf:
         return HttpResponse(pdf,content_type='application/pdf')
     else:
@@ -1262,6 +1264,9 @@ def document_bibtex(request, name, rev=None):
 
     doc = get_object_or_404(Document, name=name)
 
+    if doc.type_id not in ["rfc", "draft"]:
+        raise Http404()
+
     doi = None
     draft_became_rfc = None
     replaced_by = None
@@ -2185,22 +2190,48 @@ def idnits2_state(request, name, rev=None):
     if doc.type_id == "rfc":
         draft = doc.came_from_draft()
         if draft:
-            zero_revision = NewRevisionDocEvent.objects.filter(doc=draft,rev='00').first()
+            zero_revision = NewRevisionDocEvent.objects.filter(
+                doc=draft, rev="00"
+            ).first()
     else:
-        zero_revision = NewRevisionDocEvent.objects.filter(doc=doc,rev='00').first()
+        zero_revision = NewRevisionDocEvent.objects.filter(doc=doc, rev="00").first()
     if zero_revision:
         doc.created = zero_revision.time
     else:
-        doc.created = doc.docevent_set.order_by('-time').first().time
+        if doc.type_id == "draft":
+            if doc.became_rfc():
+                interesting_event = (
+                    doc.became_rfc()
+                    .docevent_set.filter(type="published_rfc")
+                    .order_by("-time")
+                    .first()
+                )
+            else:
+                interesting_event = doc.docevent_set.order_by(
+                    "-time"
+                ).first()  # Is taking the most _recent_ instead of the oldest event correct?
+        else:  # doc.type_id == "rfc"
+            interesting_event = (
+                doc.docevent_set.filter(type="published_rfc").order_by("-time").first()
+            )
+        doc.created = interesting_event.time
     if doc.std_level:
         doc.deststatus = doc.std_level.name
     elif doc.intended_std_level:
         doc.deststatus = doc.intended_std_level.name
     else:
         text = doc.text()
         if text:
-            parsed_draft = PlaintextDraft(text=doc.text(), source=name, name_from_source=False)
+            parsed_draft = PlaintextDraft(
+                text=doc.text(), source=name, name_from_source=False
+            )
             doc.deststatus = parsed_draft.get_status()
         else:
-            doc.deststatus="Unknown"
-    return render(request, 'doc/idnits2-state.txt', context={'doc':doc}, content_type='text/plain;charset=utf-8')    
+            doc.deststatus = "Unknown"
+    return render(
+        request,
+        "doc/idnits2-state.txt",
+        context={"doc": doc},
+        content_type="text/plain;charset=utf-8",
+    )
+

ietf/doc/views_draft.py

@@ -560,22 +560,19 @@ def to_iesg(request,name):
     if request.method == 'POST':
 
         if request.POST.get("confirm", ""): 
-
             by = request.user.person
 
             events = []
-
-            changes = []
+            def doc_event(type, by, doc, desc):
+                return DocEvent.objects.create(type=type, by=by, doc=doc, rev=doc.rev, desc=desc)
 
             if doc.get_state_slug("draft-iesg") == "idexists":
-                e = DocEvent()
-                e.type = "started_iesg_process"
-                e.by = by
-                e.doc = doc
-                e.rev = doc.rev
-                e.desc = "Document is now in IESG state <b>%s</b>" % target_state['iesg'].name
-                e.save()
-                events.append(e)
+                events.append(doc_event("started_iesg_process", by, doc, f"Document is now in IESG state <b>{target_state['iesg'].name}</b>"))
+
+            # do this first, so AD becomes action holder
+            if not doc.ad == ad :
+                doc.ad = ad
+                events.append(doc_event("changed_document", by, doc, f"Responsible AD changed to {doc.ad}"))
 
             for state_type in ['draft-iesg','draft-stream-ietf']:
                 prev_state=doc.get_state(state_type)
@@ -587,25 +584,14 @@ def to_iesg(request,name):
                         events.append(e)
                     events.append(add_state_change_event(doc=doc,by=by,prev_state=prev_state,new_state=new_state))
 
-            if not doc.ad == ad :
-                doc.ad = ad
-                changes.append("Responsible AD changed to %s" % doc.ad)
-
             if not doc.notify == notify :
                 doc.notify = notify
-                changes.append("State Change Notice email list changed to %s" % doc.notify)
+                events.append(doc_event("changed_document", by, doc, f"State Change Notice email list changed to {doc.notify}"))
 
             # Get the last available writeup
             previous_writeup = doc.latest_event(WriteupDocEvent,type="changed_protocol_writeup")
             if previous_writeup != None:
-                changes.append(previous_writeup.text)
-
-            for c in changes:
-                e = DocEvent(doc=doc, rev=doc.rev, by=by)
-                e.desc = c
-                e.type = "changed_document"
-                e.save()
-                events.append(e)
+                events.append(doc_event("changed_document", by, doc, previous_writeup.text))
 
             doc.save_with_history(events)
 

ietf/doc/views_help.py

@@ -1,5 +1,7 @@
 # Copyright The IETF Trust 2013-2023, All Rights Reserved
 
+import debug # pyflakes: ignore
+
 from django.shortcuts import render, get_object_or_404
 from django.http import Http404
 
@@ -18,6 +20,7 @@ def state_help(request, type=None):
         "draft-stream-irtf": ("draft-stream-irtf", "IRTF Stream States for Internet-Drafts"),
         "draft-stream-ise": ("draft-stream-ise", "ISE Stream States for Internet-Drafts"),
         "draft-stream-iab": ("draft-stream-iab", "IAB Stream States for Internet-Drafts"),
+        "draft-stream-editorial": ("draft-stream-editorial", "Editorial Stream States for Internet-Drafts"),
         "charter": ("charter", "Charter States"),
         "conflict-review": ("conflrev", "Conflict Review States"),
         "status-change": ("statchg", "RFC Status Change States"),

ietf/doc/views_search.py

@@ -211,6 +211,9 @@ def retrieve_search_results(form, all_types=False):
                     Q(targets_related__source__title__icontains=singlespace, targets_related__relationship_id="contains"),
                 ])
 
+        if query["rfcs"]:
+            queries.extend([Q(targets_related__source__name__icontains=look_for, targets_related__relationship_id="became_rfc")])
+
         combined_query = reduce(operator.or_, queries)
         docs = docs.filter(combined_query).distinct()
 
@@ -468,11 +471,11 @@ def ad_workload(request):
             state = doc_state(doc)
 
             state_events = doc.docevent_set.filter(
-                Q(type="started_iesg_process")
-                | Q(type="changed_state")
-                | Q(type="published_rfc")
-                | Q(type="closed_ballot"),
-            ).order_by("-time")
+                type__in=["started_iesg_process", "changed_state", "closed_ballot"]
+            )
+            if doc.became_rfc():
+                state_events = state_events | doc.became_rfc().docevent_set.filter(type="published_rfc")
+            state_events = state_events.order_by("-time")
 
             # compute state history for drafts
             last = now