From 4dd9dd131723497db3d2aa76166169fd32c42fdd Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Jan 2024 15:17:12 -0400 Subject: [PATCH 01/11] test: Test rfc_editor_index_update_task --- ietf/sync/tests.py | 618 ++++++++++++++++++++++++++++++++------------- 1 file changed, 444 insertions(+), 174 deletions(-) diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 6ac8f4afb03..18672697dbc 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -9,18 +9,29 @@ import mock import quopri +from dataclasses import dataclass + from django.conf import settings from django.urls import reverse as urlreverse from django.utils import timezone +from django.test.utils import override_settings -import debug # pyflakes:ignore +import debug # pyflakes:ignore from ietf.doc.factories import WgDraftFactory, RfcFactory -from ietf.doc.models import Document, DocEvent, DeletedEvent, DocTagName, RelatedDocument, State, StateDocEvent +from ietf.doc.models import ( + Document, + DocEvent, + DeletedEvent, + DocTagName, + RelatedDocument, + State, + StateDocEvent, +) from ietf.doc.utils import add_state_change_event from ietf.group.factories import GroupFactory from ietf.person.models import Person -from ietf.sync import iana, rfceditor +from ietf.sync import iana, rfceditor, tasks from ietf.utils.mail import outbox, empty_outbox from ietf.utils.test_utils import login_testing_unauthorized from ietf.utils.test_utils import TestCase @@ -31,25 +42,41 @@ class IANASyncTests(TestCase): def test_protocol_page_sync(self): draft = WgDraftFactory() rfc = RfcFactory(rfc_number=1234) - draft.relateddocument_set.create(relationship_id="became_rfc", target = rfc) - DocEvent.objects.create(doc=rfc, rev="", type="published_rfc", by=Person.objects.get(name="(System)")) + draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc) + DocEvent.objects.create( + doc=rfc, + rev="", + type="published_rfc", + by=Person.objects.get(name="(System)"), + ) - rfc_names = iana.parse_protocol_page('RFC 1234') + rfc_names = iana.parse_protocol_page( + 'RFC 1234' + ) self.assertEqual(len(rfc_names), 1) self.assertEqual(rfc_names[0], "rfc1234") - iana.update_rfc_log_from_protocol_page(rfc_names, timezone.now() - datetime.timedelta(days=1)) - self.assertEqual(DocEvent.objects.filter(doc=rfc, type="rfc_in_iana_registry").count(), 1) + iana.update_rfc_log_from_protocol_page( + rfc_names, timezone.now() - datetime.timedelta(days=1) + ) + self.assertEqual( + DocEvent.objects.filter(doc=rfc, type="rfc_in_iana_registry").count(), 1 + ) # make sure it doesn't create duplicates - iana.update_rfc_log_from_protocol_page(rfc_names, timezone.now() - datetime.timedelta(days=1)) - self.assertEqual(DocEvent.objects.filter(doc=rfc, type="rfc_in_iana_registry").count(), 1) + iana.update_rfc_log_from_protocol_page( + rfc_names, timezone.now() - datetime.timedelta(days=1) + ) + self.assertEqual( + DocEvent.objects.filter(doc=rfc, type="rfc_in_iana_registry").count(), 1 + ) def test_changes_sync(self): - draft = WgDraftFactory(ad=Person.objects.get(user__username='ad')) + draft = WgDraftFactory(ad=Person.objects.get(user__username="ad")) - data = json.dumps({ - "changes": [ + data = json.dumps( + { + "changes": [ { "time": "2011-10-09 12:00:01", "doc": draft.name, @@ -59,7 +86,7 @@ def test_changes_sync(self): { "time": "2011-10-09 12:00:02", "doc": draft.name, - "state": "IANA - Review Needed", # this should be skipped + "state": "IANA - Review Needed", # this should be skipped "type": "iana_review", }, { @@ -73,9 +100,10 @@ def test_changes_sync(self): "doc": draft.name, "state": "In Progress", "type": "iana_state", - } + }, ] - }) + } + ) changes = iana.parse_changes_json(data) # check sorting @@ -88,12 +116,17 @@ def test_changes_sync(self): self.assertEqual(len(warnings), 0) self.assertEqual(draft.get_state_slug("draft-iana-review"), "not-ok") self.assertEqual(draft.get_state_slug("draft-iana-action"), "waitrfc") - e = draft.latest_event(StateDocEvent, type="changed_state", state_type="draft-iana-action") - self.assertEqual(e.desc, "IANA Action state changed to Waiting on RFC Editor from In Progress") -# self.assertEqual(e.time, datetime.datetime(2011, 10, 9, 5, 0)) # check timezone handling - self.assertEqual(len(outbox), 3 ) + e = draft.latest_event( + StateDocEvent, type="changed_state", state_type="draft-iana-action" + ) + self.assertEqual( + e.desc, + "IANA Action state changed to Waiting on RFC Editor from In Progress", + ) + # self.assertEqual(e.time, datetime.datetime(2011, 10, 9, 5, 0)) # check timezone handling + self.assertEqual(len(outbox), 3) for m in outbox: - self.assertTrue('aread@' in m['To']) + self.assertTrue("aread@" in m["To"]) # make sure it doesn't create duplicates added_events, warnings = iana.update_history_with_changes(changes) @@ -104,36 +137,38 @@ def test_changes_sync_errors(self): draft = WgDraftFactory() # missing "type" - data = json.dumps({ + data = json.dumps( + { "changes": [ - { - "time": "2011-10-09 12:00:01", - "doc": draft.name, - "state": "IANA Not OK", - }, - ] - }) + { + "time": "2011-10-09 12:00:01", + "doc": draft.name, + "state": "IANA Not OK", + }, + ] + } + ) self.assertRaises(Exception, iana.parse_changes_json, data) # error response - data = json.dumps({ - "error": "I am in error." - }) + data = json.dumps({"error": "I am in error."}) self.assertRaises(Exception, iana.parse_changes_json, data) - + # missing document from database - data = json.dumps({ + data = json.dumps( + { "changes": [ - { - "time": "2011-10-09 12:00:01", - "doc": "draft-this-does-not-exist", - "state": "IANA Not OK", - "type": "iana_review", - }, - ] - }) + { + "time": "2011-10-09 12:00:01", + "doc": "draft-this-does-not-exist", + "state": "IANA Not OK", + "type": "iana_review", + }, + ] + } + ) changes = iana.parse_changes_json(data) added_events, warnings = iana.update_history_with_changes(changes) @@ -143,7 +178,7 @@ def test_changes_sync_errors(self): def test_iana_review_mail(self): draft = WgDraftFactory() - subject_template = 'Subject: [IANA #12345] Last Call: <%(draft)s-%(rev)s.txt> (Long text) to Informational RFC' + subject_template = "Subject: [IANA #12345] Last Call: <%(draft)s-%(rev)s.txt> (Long text) to Informational RFC" msg_template = """From: %(fromaddr)s Date: Thu, 10 May 2012 12:00:0%(rtime)d +0000 Content-Transfer-Encoding: quoted-printable @@ -169,79 +204,107 @@ def test_iana_review_mail(self): (END IANA %(tag)s) """ - subjects = ( subject_template % dict(draft=draft.name,rev=draft.rev) , 'Subject: Vacuous Subject' ) + subjects = ( + subject_template % dict(draft=draft.name, rev=draft.rev), + "Subject: Vacuous Subject", + ) - tags = ('LAST CALL COMMENTS', 'COMMENTS') + tags = ("LAST CALL COMMENTS", "COMMENTS") - embedded_names = (': %s-%s.txt'%(draft.name,draft.rev), '') + embedded_names = (": %s-%s.txt" % (draft.name, draft.rev), "") for subject in subjects: for tag in tags: for embedded_name in embedded_names: - if embedded_name or not 'Vacuous' in subject: - - rtime = 7*subjects.index(subject) + 5*tags.index(tag) + embedded_names.index(embedded_name) - person=Person.objects.get(user__username="iana") + if embedded_name or not "Vacuous" in subject: + rtime = ( + 7 * subjects.index(subject) + + 5 * tags.index(tag) + + embedded_names.index(embedded_name) + ) + person = Person.objects.get(user__username="iana") fromaddr = person.email().formatted_email() - msg = msg_template % dict(person=quopri.encodestring(person.name.encode('utf-8')), - fromaddr=fromaddr, - draft=draft.name, - rev=draft.rev, - tag=tag, - rtime=rtime, - subject=subject, - embedded_name=embedded_name,) - doc_name, review_time, by, comment = iana.parse_review_email(msg.encode('utf-8')) - + msg = msg_template % dict( + person=quopri.encodestring(person.name.encode("utf-8")), + fromaddr=fromaddr, + draft=draft.name, + rev=draft.rev, + tag=tag, + rtime=rtime, + subject=subject, + embedded_name=embedded_name, + ) + doc_name, review_time, by, comment = iana.parse_review_email( + msg.encode("utf-8") + ) + self.assertEqual(doc_name, draft.name) - self.assertEqual(review_time, datetime.datetime(2012, 5, 10, 12, 0, rtime, tzinfo=datetime.timezone.utc)) + self.assertEqual( + review_time, + datetime.datetime( + 2012, 5, 10, 12, 0, rtime, tzinfo=datetime.timezone.utc + ), + ) self.assertEqual(by, Person.objects.get(user__username="iana")) - self.assertIn("there are no IANA Actions", comment.replace("\n", "")) - - events_before = DocEvent.objects.filter(doc=draft, type="iana_review").count() + self.assertIn( + "there are no IANA Actions", comment.replace("\n", "") + ) + + events_before = DocEvent.objects.filter( + doc=draft, type="iana_review" + ).count() iana.add_review_comment(doc_name, review_time, by, comment) - + e = draft.latest_event(type="iana_review") self.assertTrue(e) self.assertEqual(e.desc, comment) self.assertEqual(e.by, by) - + # make sure it doesn't create duplicates iana.add_review_comment(doc_name, review_time, by, comment) - self.assertEqual(DocEvent.objects.filter(doc=draft, type="iana_review").count(), events_before+1) + self.assertEqual( + DocEvent.objects.filter( + doc=draft, type="iana_review" + ).count(), + events_before + 1, + ) def test_notify_page(self): # check that we can get the notify page - url = urlreverse("ietf.sync.views.notify", kwargs=dict(org="iana", notification="changes")) + url = urlreverse( + "ietf.sync.views.notify", kwargs=dict(org="iana", notification="changes") + ) login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertContains(r, "new changes at") # we don't actually try posting as that would trigger a real run - + class RFCSyncTests(TestCase): def write_draft_file(self, name, size): - with io.open(os.path.join(settings.INTERNET_DRAFT_PATH, name), 'w') as f: + with io.open(os.path.join(settings.INTERNET_DRAFT_PATH, name), "w") as f: f.write("a" * size) def test_rfc_index(self): - area = GroupFactory(type_id='area') + area = GroupFactory(type_id="area") draft_doc = WgDraftFactory( group__parent=area, - states=[('draft-iesg','rfcqueue')], - ad=Person.objects.get(user__username='ad'), + states=[("draft-iesg", "rfcqueue")], + ad=Person.objects.get(user__username="ad"), external_url="http://my-external-url.example.com", note="this is a note", ) - draft_doc.action_holders.add(draft_doc.ad) # not normally set, but add to be sure it's cleared + draft_doc.action_holders.add( + draft_doc.ad + ) # not normally set, but add to be sure it's cleared RfcFactory(rfc_number=123) today = date_today() - t = ''' + t = """ """ % dict( + year=today.strftime("%Y"), + month=today.strftime("%B"), + name=draft_doc.name, + rev=draft_doc.rev, + area=draft_doc.group.parent.acronym, + group=draft_doc.group.acronym, + ) + + errata = [ + { + "errata_id": 1, + "doc-id": "RFC123", # n.b. this is not the same RFC as in the above index XML! + "errata_status_code": "Verified", + "errata_type_code": "Editorial", "section": "4.1", - "orig_text":" S: 220-smtp.example.com ESMTP Server", - "correct_text":" S: 220 smtp.example.com ESMTP Server", - "notes":"There are 3 instances of this (one on p. 7 and two on p. 8). \n", - "submit_date":"2007-07-19", - "submitter_name":"Rob Siemborski", - "verifier_id":99, - "verifier_name":None, - "update_date":"2019-09-10 09:09:03"}, + "orig_text": " S: 220-smtp.example.com ESMTP Server", + "correct_text": " S: 220 smtp.example.com ESMTP Server", + "notes": "There are 3 instances of this (one on p. 7 and two on p. 8). \n", + "submit_date": "2007-07-19", + "submitter_name": "Rob Siemborski", + "verifier_id": 99, + "verifier_name": None, + "update_date": "2019-09-10 09:09:03", + }, ] data = rfceditor.parse_index(io.StringIO(t)) self.assertEqual(len(data), 1) - rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract = data[0] + ( + rfc_number, + title, + authors, + rfc_published_date, + current_status, + updates, + updated_by, + obsoletes, + obsoleted_by, + also, + draft, + has_errata, + stream, + wg, + file_formats, + pages, + abstract, + ) = data[0] # currently, we only check what we actually use self.assertEqual(rfc_number, 1234) @@ -350,8 +435,12 @@ def test_rfc_index(self): changes = [] with mock.patch("ietf.sync.rfceditor.log") as mock_log: - for rfc_number, _, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)): - changes.append({"doc_pk": d.pk, "rfc_published": rfc_published}) # we ignore the actual change list + for rfc_number, _, d, rfc_published in rfceditor.update_docs_from_rfc_index( + data, errata, today - datetime.timedelta(days=30) + ): + changes.append( + {"doc_pk": d.pk, "rfc_published": rfc_published} + ) # we ignore the actual change list self.assertEqual(rfc_number, 1234) if rfc_published: self.assertEqual(d.type_id, "rfc") @@ -359,7 +448,7 @@ def test_rfc_index(self): else: self.assertEqual(d.type_id, "draft") self.assertIsNone(d.rfc_number) - + self.assertFalse(mock_log.called, "No log messages expected") draft_doc = Document.objects.get(name=draft_doc.name) @@ -373,40 +462,88 @@ def test_rfc_index(self): self.assertEqual(draft_doc.title, draft_title_before) self.assertEqual(draft_doc.abstract, draft_abstract_before) self.assertEqual(draft_doc.pages, draft_pages_before) - self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, draft_filename))) - self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, draft_filename))) + self.assertTrue( + not os.path.exists( + os.path.join(settings.INTERNET_DRAFT_PATH, draft_filename) + ) + ) + self.assertTrue( + os.path.exists( + os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, draft_filename) + ) + ) rfc_doc = Document.objects.filter(rfc_number=1234, type_id="rfc").first() self.assertIsNotNone(rfc_doc, "RFC document should have been created") rfc_events = rfc_doc.docevent_set.all() self.assertEqual(len(rfc_events), 8) expected_events = [ - ["sync_from_rfc_editor", ""], # Not looking for exact desc match here - see detailed tests below - ["sync_from_rfc_editor", "Imported membership of rfc1234 in std2 via sync to the rfc-index"], - ["std_history_marker", "No history of STD2 is currently available in the datatracker before this point"], - ["sync_from_rfc_editor", "Imported membership of rfc1234 in fyi1 via sync to the rfc-index"], - ["fyi_history_marker", "No history of FYI1 is currently available in the datatracker before this point"], - ["sync_from_rfc_editor", "Imported membership of rfc1234 in bcp1 via sync to the rfc-index"], - ["bcp_history_marker", "No history of BCP1 is currently available in the datatracker before this point"], - ["published_rfc", "RFC published"] + [ + "sync_from_rfc_editor", + "", + ], # Not looking for exact desc match here - see detailed tests below + [ + "sync_from_rfc_editor", + "Imported membership of rfc1234 in std2 via sync to the rfc-index", + ], + [ + "std_history_marker", + "No history of STD2 is currently available in the datatracker before this point", + ], + [ + "sync_from_rfc_editor", + "Imported membership of rfc1234 in fyi1 via sync to the rfc-index", + ], + [ + "fyi_history_marker", + "No history of FYI1 is currently available in the datatracker before this point", + ], + [ + "sync_from_rfc_editor", + "Imported membership of rfc1234 in bcp1 via sync to the rfc-index", + ], + [ + "bcp_history_marker", + "No history of BCP1 is currently available in the datatracker before this point", + ], + ["published_rfc", "RFC published"], ] for index, [event_type, desc] in enumerate(expected_events): self.assertEqual(rfc_events[index].type, event_type) if index == 0: - self.assertIn("Received changes through RFC Editor sync (created document RFC 1234,", rfc_events[0].desc) - self.assertIn(f"created became rfc relationship between {rfc_doc.came_from_draft().name} and RFC 1234", rfc_events[0].desc) + self.assertIn( + "Received changes through RFC Editor sync (created document RFC 1234,", + rfc_events[0].desc, + ) + self.assertIn( + f"created became rfc relationship between {rfc_doc.came_from_draft().name} and RFC 1234", + rfc_events[0].desc, + ) self.assertIn("set title to 'A Testing RFC'", rfc_events[0].desc) - self.assertIn("set abstract to 'This is some interesting text.'", rfc_events[0].desc) + self.assertIn( + "set abstract to 'This is some interesting text.'", + rfc_events[0].desc, + ) self.assertIn("set pages to 42", rfc_events[0].desc) - self.assertIn("set standardization level to Proposed Standard", rfc_events[0].desc) - self.assertIn(f"added RFC published event at {rfc_events[0].time.astimezone(RPC_TZINFO):%Y-%m-%d}", rfc_events[0].desc) - self.assertIn("created updates relation between RFC 1234 and RFC 123", rfc_events[0].desc) + self.assertIn( + "set standardization level to Proposed Standard", rfc_events[0].desc + ) + self.assertIn( + f"added RFC published event at {rfc_events[0].time.astimezone(RPC_TZINFO):%Y-%m-%d}", + rfc_events[0].desc, + ) + self.assertIn( + "created updates relation between RFC 1234 and RFC 123", + rfc_events[0].desc, + ) self.assertIn("added Errata tag", rfc_events[0].desc) else: self.assertEqual(rfc_events[index].desc, desc) self.assertEqual(rfc_events[7].time.astimezone(RPC_TZINFO).date(), today) for subseries_name in ["bcp1", "fyi1", "std2"]: - sub = Document.objects.filter(type_id=subseries_name[:3],name=subseries_name).first() + sub = Document.objects.filter( + type_id=subseries_name[:3], name=subseries_name + ).first() self.assertIsNotNone(sub, f"{subseries_name} not created") self.assertTrue(rfc_doc in sub.contains()) self.assertTrue(sub in rfc_doc.part_of()) @@ -421,8 +558,16 @@ def test_rfc_index(self): # self.assertTrue(DocAlias.objects.filter(name="bcp1", docs=rfc_doc)) # self.assertTrue(DocAlias.objects.filter(name="fyi1", docs=rfc_doc)) # self.assertTrue(DocAlias.objects.filter(name="std1", docs=rfc_doc)) - self.assertTrue(RelatedDocument.objects.filter(source=rfc_doc, target__name="rfc123", relationship="updates").exists()) - self.assertTrue(RelatedDocument.objects.filter(source=draft_doc, target=rfc_doc, relationship="became_rfc").exists()) + self.assertTrue( + RelatedDocument.objects.filter( + source=rfc_doc, target__name="rfc123", relationship="updates" + ).exists() + ) + self.assertTrue( + RelatedDocument.objects.filter( + source=draft_doc, target=rfc_doc, relationship="became_rfc" + ).exists() + ) self.assertEqual(rfc_doc.title, "A Testing RFC") self.assertEqual(rfc_doc.abstract, "This is some interesting text.") self.assertEqual(rfc_doc.std_level_id, "ps") @@ -442,12 +587,16 @@ def test_rfc_index(self): self.assertEqual(changes[1]["rfc_published"], True) # make sure we can apply it again with no changes - changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30))) + changed = list( + rfceditor.update_docs_from_rfc_index( + data, errata, today - datetime.timedelta(days=30) + ) + ) self.assertEqual(len(changed), 0) def _generate_rfc_queue_xml(self, draft, state, auth48_url=None): """Generate an RFC queue xml string for a draft""" - t = ''' + t = """
%(name)s-%(rev)s.txt @@ -466,24 +615,30 @@ def _generate_rfc_queue_xml(self, draft, state, auth48_url=None): %(group)s
-''' % dict(name=draft.name, - rev=draft.rev, - title=draft.title, - group=draft.group.name, - ref="draft-ietf-test", - state=state, - auth48_url=(auth48_url or '')) - t = t.replace('\n', '') # strip empty auth48-url tags +""" % dict( + name=draft.name, + rev=draft.rev, + title=draft.title, + group=draft.group.name, + ref="draft-ietf-test", + state=state, + auth48_url=(auth48_url or ""), + ) + t = t.replace("\n", "") # strip empty auth48-url tags return t def test_rfc_queue(self): - draft = WgDraftFactory(states=[('draft-iesg','ann')], ad=Person.objects.get(user__username='ad')) - draft.action_holders.add(draft.ad) # add an action holder so we can test that it's removed later + draft = WgDraftFactory( + states=[("draft-iesg", "ann")], ad=Person.objects.get(user__username="ad") + ) + draft.action_holders.add( + draft.ad + ) # add an action holder so we can test that it's removed later expected_auth48_url = "http://www.rfc-editor.org/auth48/rfc1234" - t = self._generate_rfc_queue_xml(draft, - state='EDIT*R*A(1G)', - auth48_url=expected_auth48_url) + t = self._generate_rfc_queue_xml( + draft, state="EDIT*R*A(1G)", auth48_url=expected_auth48_url + ) drafts, warnings = rfceditor.parse_queue(io.StringIO(t)) # rfceditor.parse_queue() is tested independently; just sanity check here @@ -500,11 +655,16 @@ def test_rfc_queue(self): self.assertEqual(draft.get_state_slug("draft-rfceditor"), "edit") self.assertEqual(draft.get_state_slug("draft-iesg"), "rfcqueue") self.assertCountEqual(draft.action_holders.all(), []) - self.assertEqual(set(draft.tags.all()), set(DocTagName.objects.filter(slug__in=("iana", "ref")))) + self.assertEqual( + set(draft.tags.all()), + set(DocTagName.objects.filter(slug__in=("iana", "ref"))), + ) events = draft.docevent_set.all() - self.assertEqual(events[0].type, "changed_state") # changed draft-iesg state + self.assertEqual(events[0].type, "changed_state") # changed draft-iesg state self.assertEqual(events[1].type, "changed_action_holders") - self.assertEqual(events[2].type, "changed_state") # changed draft-rfceditor state + self.assertEqual( + events[2].type, "changed_state" + ) # changed draft-rfceditor state self.assertEqual(events[3].type, "rfc_editor_received_announcement") self.assertEqual(len(outbox), mailbox_before + 1) @@ -518,19 +678,31 @@ def test_rfc_queue(self): def test_rfceditor_parse_queue(self): """Test that rfceditor.parse_queue() behaves as expected. - Currently does a limited test - old comment was + Currently does a limited test - old comment was "currently, we only check what we actually use". """ - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - t = self._generate_rfc_queue_xml(draft, - state='EDIT*R*A(1G)', - auth48_url="http://www.rfc-editor.org/auth48/rfc1234") + draft = WgDraftFactory(states=[("draft-iesg", "ann")]) + t = self._generate_rfc_queue_xml( + draft, + state="EDIT*R*A(1G)", + auth48_url="http://www.rfc-editor.org/auth48/rfc1234", + ) drafts, warnings = rfceditor.parse_queue(io.StringIO(t)) self.assertEqual(len(drafts), 1) self.assertEqual(len(warnings), 0) - draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs = drafts[0] + ( + draft_name, + date_received, + state, + tags, + missref_generation, + stream, + auth48, + cluster, + refs, + ) = drafts[0] self.assertEqual(draft_name, draft.name) self.assertEqual(state, "EDIT") self.assertEqual(set(tags), set(["iana", "ref"])) @@ -538,63 +710,66 @@ def test_rfceditor_parse_queue(self): def test_rfceditor_parse_queue_TI_state(self): # Test with TI state introduced 11 Sep 2019 - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - t = self._generate_rfc_queue_xml(draft, - state='TI', - auth48_url="http://www.rfc-editor.org/auth48/rfc1234") + draft = WgDraftFactory(states=[("draft-iesg", "ann")]) + t = self._generate_rfc_queue_xml( + draft, state="TI", auth48_url="http://www.rfc-editor.org/auth48/rfc1234" + ) __, warnings = rfceditor.parse_queue(io.StringIO(t)) self.assertEqual(len(warnings), 0) def _generate_rfceditor_update(self, draft, state, tags=None, auth48_url=None): """Helper to generate fake output from rfceditor.parse_queue()""" - return [[ - draft.name, # draft_name - '2020-06-03', # date_received - state, - tags or [], - '1', # missref_generation - 'ietf', # stream - auth48_url or '', - '', # cluster - ['draft-ietf-test'], # refs - ]] + return [ + [ + draft.name, # draft_name + "2020-06-03", # date_received + state, + tags or [], + "1", # missref_generation + "ietf", # stream + auth48_url or "", + "", # cluster + ["draft-ietf-test"], # refs + ] + ] def test_update_draft_auth48_url(self): """Test that auth48 URLs are handled correctly.""" - draft = WgDraftFactory(states=[('draft-iesg','ann')]) + draft = WgDraftFactory(states=[("draft-iesg", "ann")]) # Step 1 setup: update to a state with no auth48 URL changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='EDIT') + self._generate_rfceditor_update(draft, state="EDIT") ) self.assertEqual(len(changed), 1) self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() + auth48_docurl = draft.documenturl_set.filter(tag_id="auth48").first() self.assertIsNone(auth48_docurl) # Step 2: update to auth48 state with auth48 URL changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='AUTH48', auth48_url='http://www.rfc-editor.org/rfc1234') + self._generate_rfceditor_update( + draft, state="AUTH48", auth48_url="http://www.rfc-editor.org/rfc1234" + ) ) self.assertEqual(len(changed), 1) self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() + auth48_docurl = draft.documenturl_set.filter(tag_id="auth48").first() self.assertIsNotNone(auth48_docurl) - self.assertEqual(auth48_docurl.url, 'http://www.rfc-editor.org/rfc1234') + self.assertEqual(auth48_docurl.url, "http://www.rfc-editor.org/rfc1234") # Step 3: update to auth48-done state without auth48 URL changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='AUTH48-DONE') + self._generate_rfceditor_update(draft, state="AUTH48-DONE") ) self.assertEqual(len(changed), 1) self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() + auth48_docurl = draft.documenturl_set.filter(tag_id="auth48").first() self.assertIsNone(auth48_docurl) class DiscrepanciesTests(TestCase): def test_discrepancies(self): - # draft approved but no RFC Editor state doc = Document.objects.create(name="draft-ietf-test1", type_id="draft") doc.set_state(State.objects.get(used=True, type="draft-iesg", slug="ann")) @@ -605,7 +780,9 @@ def test_discrepancies(self): # draft with IANA state "In Progress" but RFC Editor state not IANA doc = Document.objects.create(name="draft-ietf-test2", type_id="draft") doc.set_state(State.objects.get(used=True, type="draft-iesg", slug="rfcqueue")) - doc.set_state(State.objects.get(used=True, type="draft-iana-action", slug="inprog")) + doc.set_state( + State.objects.get(used=True, type="draft-iana-action", slug="inprog") + ) doc.set_state(State.objects.get(used=True, type="draft-rfceditor", slug="auth")) r = self.client.get(urlreverse("ietf.sync.views.discrepancies")) @@ -615,7 +792,9 @@ def test_discrepancies(self): # but RFC Editor state is IANA doc = Document.objects.create(name="draft-ietf-test3", type_id="draft") doc.set_state(State.objects.get(used=True, type="draft-iesg", slug="rfcqueue")) - doc.set_state(State.objects.get(used=True, type="draft-iana-action", slug="waitrfc")) + doc.set_state( + State.objects.get(used=True, type="draft-iana-action", slug="waitrfc") + ) doc.set_state(State.objects.get(used=True, type="draft-rfceditor", slug="iana")) r = self.client.get(urlreverse("ietf.sync.views.discrepancies")) @@ -630,21 +809,30 @@ def test_discrepancies(self): r = self.client.get(urlreverse("ietf.sync.views.discrepancies")) self.assertContains(r, doc.name) + class RFCEditorUndoTests(TestCase): def test_rfceditor_undo(self): draft = WgDraftFactory() - e1 = add_state_change_event(draft, Person.objects.get(name="(System)"), None, - State.objects.get(used=True, type="draft-rfceditor", slug="auth")) + e1 = add_state_change_event( + draft, + Person.objects.get(name="(System)"), + None, + State.objects.get(used=True, type="draft-rfceditor", slug="auth"), + ) e1.desc = "First" e1.save() - e2 = add_state_change_event(draft, Person.objects.get(name="(System)"), None, - State.objects.get(used=True, type="draft-rfceditor", slug="edit")) + e2 = add_state_change_event( + draft, + Person.objects.get(name="(System)"), + None, + State.objects.get(used=True, type="draft-rfceditor", slug="edit"), + ) e2.desc = "Second" e2.save() - - url = urlreverse('ietf.sync.views.rfceditor_undo') + + url = urlreverse("ietf.sync.views.rfceditor_undo") login_testing_unauthorized(self, "rfc", url) # get @@ -672,3 +860,85 @@ def test_rfceditor_undo(self): e.content_type.model_class().objects.create(**json.loads(e.json)) self.assertTrue(StateDocEvent.objects.filter(desc="First", doc=draft)) + + +class TaskTests(TestCase): + @override_settings( + RFC_EDITOR_INDEX_URL="https://rfc-editor.example.com/index/", + RFC_EDITOR_ERRATA_JSON_URL="https://rfc-editor.example.com/errata/", + ) + @mock.patch("ietf.sync.tasks.update_docs_from_rfc_index") + @mock.patch("ietf.sync.tasks.parse_index") + @mock.patch("ietf.sync.tasks.requests.get") + def test_rfc_editor_index_update_task( + self, requests_get_mock, parse_index_mock, update_docs_mock + ): + @dataclass + class MockIndexData: + """Mock index item that claims to be a specified length""" + length: int + + def __len__(self): + return self.length + + @dataclass + class MockResponse: + """Mock object that contains text and json() that claims to be a specified length""" + text: str + json_length: int = 0 + + def json(self): + return MockIndexData(length=self.json_length) + + # Response objects + index_response = MockResponse(text="this is the index") + errata_response = MockResponse( + text="these are the errata", json_length=rfceditor.MIN_ERRATA_RESULTS + ) + + # Test with full_index = False + requests_get_mock.side_effect = (index_response, errata_response) # will step through these + parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) + update_docs_mock.return_value = [] # not tested + + tasks.rfc_editor_index_update_task(full_index=False) + + # Check parse_index() call + self.assertTrue(parse_index_mock.called) + (parse_index_args, _) = parse_index_mock.call_args + self.assertEqual( + parse_index_args[0].read(), # arg is a StringIO + "this is the index", + "parse_index is called with the index text in a StringIO", + ) + + # Check update_docs_from_rfc_index call + self.assertTrue(update_docs_mock.called) + (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args + self.assertEqual( + update_docs_args, (parse_index_mock.return_value, errata_response.json()) + ) + self.assertIsNotNone(update_docs_kwargs["skip_older_than_date"]) + + # Test again with full_index = True + requests_get_mock.side_effect = (index_response, errata_response) # will step through these + parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) + update_docs_mock.return_value = [] # not tested + tasks.rfc_editor_index_update_task(full_index=True) + + # Check parse_index() call + self.assertTrue(parse_index_mock.called) + (parse_index_args, _) = parse_index_mock.call_args + self.assertEqual( + parse_index_args[0].read(), # arg is a StringIO + "this is the index", + "parse_index is called with the index text in a StringIO", + ) + + # Check update_docs_from_rfc_index call + self.assertTrue(update_docs_mock.called) + (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args + self.assertEqual( + update_docs_args, (parse_index_mock.return_value, errata_response.json()) + ) + self.assertIsNone(update_docs_kwargs["skip_older_than_date"]) From 0395410d665c0d310248fd151386f013357c5d13 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Jan 2024 15:25:35 -0400 Subject: [PATCH 02/11] chore: Add docstring to test --- ietf/sync/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 18672697dbc..f2ae654bda6 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -873,6 +873,11 @@ class TaskTests(TestCase): def test_rfc_editor_index_update_task( self, requests_get_mock, parse_index_mock, update_docs_mock ): + """rfc_editor_index_update_task calls helpers correctly + + This tests that data flow is as expected. Assumes the individual helpers are + separately tested to function correctly. + """ @dataclass class MockIndexData: """Mock index item that claims to be a specified length""" From 9caa1948e5bc3ce7b776aee0e1c2ec63891d78b7 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Jan 2024 16:14:42 -0400 Subject: [PATCH 03/11] fix: Reuse stats instead of fetching twice --- ietf/stats/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ietf/stats/tasks.py b/ietf/stats/tasks.py index 5f51285b4f4..808e797a40b 100644 --- a/ietf/stats/tasks.py +++ b/ietf/stats/tasks.py @@ -19,9 +19,9 @@ def fetch_meeting_attendance_task(): except RuntimeError as err: log.log(f"Error in fetch_meeting_attendance_task: {err}") else: - for meeting, stats in zip(meetings, fetch_attendance_from_meetings(meetings)): + for meeting, meeting_stats in zip(meetings, stats): log.log( "Fetched data for meeting {:>3}: {:4d} processed, {:4d} added, {:4d} in table".format( - meeting.number, stats.processed, stats.added, stats.total + meeting.number, meeting_stats.processed, meeting_stats.added, meeting_stats.total ) ) From b504f4e97d316295bb4a374a5614680462f444c1 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Jan 2024 16:21:49 -0400 Subject: [PATCH 04/11] test: Test fetch_meeting_attendance_task --- ietf/stats/tests.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 5f23b1b0a0c..e4194a7f92b 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -29,7 +29,8 @@ from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory from ietf.stats.models import MeetingRegistration, CountryAlias from ietf.stats.factories import MeetingRegistrationFactory -from ietf.stats.utils import get_meeting_registration_data +from ietf.stats.tasks import fetch_meeting_attendance_task +from ietf.stats.utils import get_meeting_registration_data, FetchStats from ietf.utils.timezone import date_today @@ -300,3 +301,20 @@ def test_get_meeting_registration_data_duplicates(self, mock_get): get_meeting_registration_data(meeting) query = MeetingRegistration.objects.all() self.assertEqual(query.count(), 2) + + +class TaskTests(TestCase): + @patch("ietf.stats.tasks.fetch_attendance_from_meetings") + def test_fetch_meeting_attendance_task(self, mock_fetch_attendance): + today = date_today() + meetings = [ + MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=1)), + MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=2)), + MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=3)), + ] + mock_fetch_attendance.return_value = [FetchStats(1,2,3), FetchStats(1,2,3)] + + fetch_meeting_attendance_task() + + self.assertEqual(mock_fetch_attendance.call_count, 1) + self.assertCountEqual(mock_fetch_attendance.call_args[0][0], meetings[0:2]) From 3b0f7692345947b670e87b5fea33d1a301fe9ac0 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Jan 2024 16:23:28 -0400 Subject: [PATCH 05/11] chore: Remove outdated tasks --- ietf/utils/tasks.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/ietf/utils/tasks.py b/ietf/utils/tasks.py index e214208cd2d..e889f08720f 100644 --- a/ietf/utils/tasks.py +++ b/ietf/utils/tasks.py @@ -14,26 +14,6 @@ from ietf.utils.mail import log_smtp_exception, send_error_email -@shared_task -def every_15m_task(): - """Queue four-times-hourly tasks for execution""" - # todo decide whether we want this to be a meta-task or to individually schedule the tasks - send_scheduled_mail_task.delay() - # Parse the last year of RFC index data to get new RFCs. Needed until - # https://github.com/ietf-tools/datatracker/issues/3734 is addressed. - rfc_editor_index_update_task.delay(full_index=False) - - -@shared_task -def daily_task(): - """Queue daily tasks for execution""" - fetch_meeting_attendance_task.delay() - send_review_reminders_task.delay() - # Run an extended version of the rfc editor update to catch changes - # with backdated timestamps - rfc_editor_index_update_task.delay(full_index=True) - - @shared_task def send_scheduled_mail_task(): """Send scheduled email From 45d7589ad6d336001d339fab35bf8073eeaa3b5f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Jan 2024 16:39:14 -0400 Subject: [PATCH 06/11] Revert "chore: Add docstring to test" This reverts commit 0395410d665c0d310248fd151386f013357c5d13. --- ietf/sync/tests.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index f2ae654bda6..18672697dbc 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -873,11 +873,6 @@ class TaskTests(TestCase): def test_rfc_editor_index_update_task( self, requests_get_mock, parse_index_mock, update_docs_mock ): - """rfc_editor_index_update_task calls helpers correctly - - This tests that data flow is as expected. Assumes the individual helpers are - separately tested to function correctly. - """ @dataclass class MockIndexData: """Mock index item that claims to be a specified length""" From 3a8bc1cb44b0a792ad9e0f0d006fe7d4a5e64eac Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Jan 2024 16:39:14 -0400 Subject: [PATCH 07/11] Revert "test: Test rfc_editor_index_update_task" This reverts commit 4dd9dd131723497db3d2aa76166169fd32c42fdd. --- ietf/sync/tests.py | 618 +++++++++++++-------------------------------- 1 file changed, 174 insertions(+), 444 deletions(-) diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 18672697dbc..6ac8f4afb03 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -9,29 +9,18 @@ import mock import quopri -from dataclasses import dataclass - from django.conf import settings from django.urls import reverse as urlreverse from django.utils import timezone -from django.test.utils import override_settings -import debug # pyflakes:ignore +import debug # pyflakes:ignore from ietf.doc.factories import WgDraftFactory, RfcFactory -from ietf.doc.models import ( - Document, - DocEvent, - DeletedEvent, - DocTagName, - RelatedDocument, - State, - StateDocEvent, -) +from ietf.doc.models import Document, DocEvent, DeletedEvent, DocTagName, RelatedDocument, State, StateDocEvent from ietf.doc.utils import add_state_change_event from ietf.group.factories import GroupFactory from ietf.person.models import Person -from ietf.sync import iana, rfceditor, tasks +from ietf.sync import iana, rfceditor from ietf.utils.mail import outbox, empty_outbox from ietf.utils.test_utils import login_testing_unauthorized from ietf.utils.test_utils import TestCase @@ -42,41 +31,25 @@ class IANASyncTests(TestCase): def test_protocol_page_sync(self): draft = WgDraftFactory() rfc = RfcFactory(rfc_number=1234) - draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc) - DocEvent.objects.create( - doc=rfc, - rev="", - type="published_rfc", - by=Person.objects.get(name="(System)"), - ) + draft.relateddocument_set.create(relationship_id="became_rfc", target = rfc) + DocEvent.objects.create(doc=rfc, rev="", type="published_rfc", by=Person.objects.get(name="(System)")) - rfc_names = iana.parse_protocol_page( - 'RFC 1234' - ) + rfc_names = iana.parse_protocol_page('RFC 1234') self.assertEqual(len(rfc_names), 1) self.assertEqual(rfc_names[0], "rfc1234") - iana.update_rfc_log_from_protocol_page( - rfc_names, timezone.now() - datetime.timedelta(days=1) - ) - self.assertEqual( - DocEvent.objects.filter(doc=rfc, type="rfc_in_iana_registry").count(), 1 - ) + iana.update_rfc_log_from_protocol_page(rfc_names, timezone.now() - datetime.timedelta(days=1)) + self.assertEqual(DocEvent.objects.filter(doc=rfc, type="rfc_in_iana_registry").count(), 1) # make sure it doesn't create duplicates - iana.update_rfc_log_from_protocol_page( - rfc_names, timezone.now() - datetime.timedelta(days=1) - ) - self.assertEqual( - DocEvent.objects.filter(doc=rfc, type="rfc_in_iana_registry").count(), 1 - ) + iana.update_rfc_log_from_protocol_page(rfc_names, timezone.now() - datetime.timedelta(days=1)) + self.assertEqual(DocEvent.objects.filter(doc=rfc, type="rfc_in_iana_registry").count(), 1) def test_changes_sync(self): - draft = WgDraftFactory(ad=Person.objects.get(user__username="ad")) + draft = WgDraftFactory(ad=Person.objects.get(user__username='ad')) - data = json.dumps( - { - "changes": [ + data = json.dumps({ + "changes": [ { "time": "2011-10-09 12:00:01", "doc": draft.name, @@ -86,7 +59,7 @@ def test_changes_sync(self): { "time": "2011-10-09 12:00:02", "doc": draft.name, - "state": "IANA - Review Needed", # this should be skipped + "state": "IANA - Review Needed", # this should be skipped "type": "iana_review", }, { @@ -100,10 +73,9 @@ def test_changes_sync(self): "doc": draft.name, "state": "In Progress", "type": "iana_state", - }, + } ] - } - ) + }) changes = iana.parse_changes_json(data) # check sorting @@ -116,17 +88,12 @@ def test_changes_sync(self): self.assertEqual(len(warnings), 0) self.assertEqual(draft.get_state_slug("draft-iana-review"), "not-ok") self.assertEqual(draft.get_state_slug("draft-iana-action"), "waitrfc") - e = draft.latest_event( - StateDocEvent, type="changed_state", state_type="draft-iana-action" - ) - self.assertEqual( - e.desc, - "IANA Action state changed to Waiting on RFC Editor from In Progress", - ) - # self.assertEqual(e.time, datetime.datetime(2011, 10, 9, 5, 0)) # check timezone handling - self.assertEqual(len(outbox), 3) + e = draft.latest_event(StateDocEvent, type="changed_state", state_type="draft-iana-action") + self.assertEqual(e.desc, "IANA Action state changed to Waiting on RFC Editor from In Progress") +# self.assertEqual(e.time, datetime.datetime(2011, 10, 9, 5, 0)) # check timezone handling + self.assertEqual(len(outbox), 3 ) for m in outbox: - self.assertTrue("aread@" in m["To"]) + self.assertTrue('aread@' in m['To']) # make sure it doesn't create duplicates added_events, warnings = iana.update_history_with_changes(changes) @@ -137,38 +104,36 @@ def test_changes_sync_errors(self): draft = WgDraftFactory() # missing "type" - data = json.dumps( - { + data = json.dumps({ "changes": [ - { - "time": "2011-10-09 12:00:01", - "doc": draft.name, - "state": "IANA Not OK", - }, - ] - } - ) + { + "time": "2011-10-09 12:00:01", + "doc": draft.name, + "state": "IANA Not OK", + }, + ] + }) self.assertRaises(Exception, iana.parse_changes_json, data) # error response - data = json.dumps({"error": "I am in error."}) + data = json.dumps({ + "error": "I am in error." + }) self.assertRaises(Exception, iana.parse_changes_json, data) - + # missing document from database - data = json.dumps( - { + data = json.dumps({ "changes": [ - { - "time": "2011-10-09 12:00:01", - "doc": "draft-this-does-not-exist", - "state": "IANA Not OK", - "type": "iana_review", - }, - ] - } - ) + { + "time": "2011-10-09 12:00:01", + "doc": "draft-this-does-not-exist", + "state": "IANA Not OK", + "type": "iana_review", + }, + ] + }) changes = iana.parse_changes_json(data) added_events, warnings = iana.update_history_with_changes(changes) @@ -178,7 +143,7 @@ def test_changes_sync_errors(self): def test_iana_review_mail(self): draft = WgDraftFactory() - subject_template = "Subject: [IANA #12345] Last Call: <%(draft)s-%(rev)s.txt> (Long text) to Informational RFC" + subject_template = 'Subject: [IANA #12345] Last Call: <%(draft)s-%(rev)s.txt> (Long text) to Informational RFC' msg_template = """From: %(fromaddr)s Date: Thu, 10 May 2012 12:00:0%(rtime)d +0000 Content-Transfer-Encoding: quoted-printable @@ -204,107 +169,79 @@ def test_iana_review_mail(self): (END IANA %(tag)s) """ - subjects = ( - subject_template % dict(draft=draft.name, rev=draft.rev), - "Subject: Vacuous Subject", - ) + subjects = ( subject_template % dict(draft=draft.name,rev=draft.rev) , 'Subject: Vacuous Subject' ) - tags = ("LAST CALL COMMENTS", "COMMENTS") + tags = ('LAST CALL COMMENTS', 'COMMENTS') - embedded_names = (": %s-%s.txt" % (draft.name, draft.rev), "") + embedded_names = (': %s-%s.txt'%(draft.name,draft.rev), '') for subject in subjects: for tag in tags: for embedded_name in embedded_names: - if embedded_name or not "Vacuous" in subject: - rtime = ( - 7 * subjects.index(subject) - + 5 * tags.index(tag) - + embedded_names.index(embedded_name) - ) - person = Person.objects.get(user__username="iana") + if embedded_name or not 'Vacuous' in subject: + + rtime = 7*subjects.index(subject) + 5*tags.index(tag) + embedded_names.index(embedded_name) + person=Person.objects.get(user__username="iana") fromaddr = person.email().formatted_email() - msg = msg_template % dict( - person=quopri.encodestring(person.name.encode("utf-8")), - fromaddr=fromaddr, - draft=draft.name, - rev=draft.rev, - tag=tag, - rtime=rtime, - subject=subject, - embedded_name=embedded_name, - ) - doc_name, review_time, by, comment = iana.parse_review_email( - msg.encode("utf-8") - ) - + msg = msg_template % dict(person=quopri.encodestring(person.name.encode('utf-8')), + fromaddr=fromaddr, + draft=draft.name, + rev=draft.rev, + tag=tag, + rtime=rtime, + subject=subject, + embedded_name=embedded_name,) + doc_name, review_time, by, comment = iana.parse_review_email(msg.encode('utf-8')) + self.assertEqual(doc_name, draft.name) - self.assertEqual( - review_time, - datetime.datetime( - 2012, 5, 10, 12, 0, rtime, tzinfo=datetime.timezone.utc - ), - ) + self.assertEqual(review_time, datetime.datetime(2012, 5, 10, 12, 0, rtime, tzinfo=datetime.timezone.utc)) self.assertEqual(by, Person.objects.get(user__username="iana")) - self.assertIn( - "there are no IANA Actions", comment.replace("\n", "") - ) - - events_before = DocEvent.objects.filter( - doc=draft, type="iana_review" - ).count() + self.assertIn("there are no IANA Actions", comment.replace("\n", "")) + + events_before = DocEvent.objects.filter(doc=draft, type="iana_review").count() iana.add_review_comment(doc_name, review_time, by, comment) - + e = draft.latest_event(type="iana_review") self.assertTrue(e) self.assertEqual(e.desc, comment) self.assertEqual(e.by, by) - + # make sure it doesn't create duplicates iana.add_review_comment(doc_name, review_time, by, comment) - self.assertEqual( - DocEvent.objects.filter( - doc=draft, type="iana_review" - ).count(), - events_before + 1, - ) + self.assertEqual(DocEvent.objects.filter(doc=draft, type="iana_review").count(), events_before+1) def test_notify_page(self): # check that we can get the notify page - url = urlreverse( - "ietf.sync.views.notify", kwargs=dict(org="iana", notification="changes") - ) + url = urlreverse("ietf.sync.views.notify", kwargs=dict(org="iana", notification="changes")) login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertContains(r, "new changes at") # we don't actually try posting as that would trigger a real run - + class RFCSyncTests(TestCase): def write_draft_file(self, name, size): - with io.open(os.path.join(settings.INTERNET_DRAFT_PATH, name), "w") as f: + with io.open(os.path.join(settings.INTERNET_DRAFT_PATH, name), 'w') as f: f.write("a" * size) def test_rfc_index(self): - area = GroupFactory(type_id="area") + area = GroupFactory(type_id='area') draft_doc = WgDraftFactory( group__parent=area, - states=[("draft-iesg", "rfcqueue")], - ad=Person.objects.get(user__username="ad"), + states=[('draft-iesg','rfcqueue')], + ad=Person.objects.get(user__username='ad'), external_url="http://my-external-url.example.com", note="this is a note", ) - draft_doc.action_holders.add( - draft_doc.ad - ) # not normally set, but add to be sure it's cleared + draft_doc.action_holders.add(draft_doc.ad) # not normally set, but add to be sure it's cleared RfcFactory(rfc_number=123) today = date_today() - t = """ + t = ''' ''' % dict(year=today.strftime("%Y"), + month=today.strftime("%B"), + name=draft_doc.name, + rev=draft_doc.rev, + area=draft_doc.group.parent.acronym, + group=draft_doc.group.acronym) + + errata = [{ + "errata_id":1, + "doc-id":"RFC123", # n.b. this is not the same RFC as in the above index XML! + "errata_status_code":"Verified", + "errata_type_code":"Editorial", "section": "4.1", - "orig_text": " S: 220-smtp.example.com ESMTP Server", - "correct_text": " S: 220 smtp.example.com ESMTP Server", - "notes": "There are 3 instances of this (one on p. 7 and two on p. 8). \n", - "submit_date": "2007-07-19", - "submitter_name": "Rob Siemborski", - "verifier_id": 99, - "verifier_name": None, - "update_date": "2019-09-10 09:09:03", - }, + "orig_text":" S: 220-smtp.example.com ESMTP Server", + "correct_text":" S: 220 smtp.example.com ESMTP Server", + "notes":"There are 3 instances of this (one on p. 7 and two on p. 8). \n", + "submit_date":"2007-07-19", + "submitter_name":"Rob Siemborski", + "verifier_id":99, + "verifier_name":None, + "update_date":"2019-09-10 09:09:03"}, ] data = rfceditor.parse_index(io.StringIO(t)) self.assertEqual(len(data), 1) - ( - rfc_number, - title, - authors, - rfc_published_date, - current_status, - updates, - updated_by, - obsoletes, - obsoleted_by, - also, - draft, - has_errata, - stream, - wg, - file_formats, - pages, - abstract, - ) = data[0] + rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract = data[0] # currently, we only check what we actually use self.assertEqual(rfc_number, 1234) @@ -435,12 +350,8 @@ def test_rfc_index(self): changes = [] with mock.patch("ietf.sync.rfceditor.log") as mock_log: - for rfc_number, _, d, rfc_published in rfceditor.update_docs_from_rfc_index( - data, errata, today - datetime.timedelta(days=30) - ): - changes.append( - {"doc_pk": d.pk, "rfc_published": rfc_published} - ) # we ignore the actual change list + for rfc_number, _, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)): + changes.append({"doc_pk": d.pk, "rfc_published": rfc_published}) # we ignore the actual change list self.assertEqual(rfc_number, 1234) if rfc_published: self.assertEqual(d.type_id, "rfc") @@ -448,7 +359,7 @@ def test_rfc_index(self): else: self.assertEqual(d.type_id, "draft") self.assertIsNone(d.rfc_number) - + self.assertFalse(mock_log.called, "No log messages expected") draft_doc = Document.objects.get(name=draft_doc.name) @@ -462,88 +373,40 @@ def test_rfc_index(self): self.assertEqual(draft_doc.title, draft_title_before) self.assertEqual(draft_doc.abstract, draft_abstract_before) self.assertEqual(draft_doc.pages, draft_pages_before) - self.assertTrue( - not os.path.exists( - os.path.join(settings.INTERNET_DRAFT_PATH, draft_filename) - ) - ) - self.assertTrue( - os.path.exists( - os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, draft_filename) - ) - ) + self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, draft_filename))) + self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, draft_filename))) rfc_doc = Document.objects.filter(rfc_number=1234, type_id="rfc").first() self.assertIsNotNone(rfc_doc, "RFC document should have been created") rfc_events = rfc_doc.docevent_set.all() self.assertEqual(len(rfc_events), 8) expected_events = [ - [ - "sync_from_rfc_editor", - "", - ], # Not looking for exact desc match here - see detailed tests below - [ - "sync_from_rfc_editor", - "Imported membership of rfc1234 in std2 via sync to the rfc-index", - ], - [ - "std_history_marker", - "No history of STD2 is currently available in the datatracker before this point", - ], - [ - "sync_from_rfc_editor", - "Imported membership of rfc1234 in fyi1 via sync to the rfc-index", - ], - [ - "fyi_history_marker", - "No history of FYI1 is currently available in the datatracker before this point", - ], - [ - "sync_from_rfc_editor", - "Imported membership of rfc1234 in bcp1 via sync to the rfc-index", - ], - [ - "bcp_history_marker", - "No history of BCP1 is currently available in the datatracker before this point", - ], - ["published_rfc", "RFC published"], + ["sync_from_rfc_editor", ""], # Not looking for exact desc match here - see detailed tests below + ["sync_from_rfc_editor", "Imported membership of rfc1234 in std2 via sync to the rfc-index"], + ["std_history_marker", "No history of STD2 is currently available in the datatracker before this point"], + ["sync_from_rfc_editor", "Imported membership of rfc1234 in fyi1 via sync to the rfc-index"], + ["fyi_history_marker", "No history of FYI1 is currently available in the datatracker before this point"], + ["sync_from_rfc_editor", "Imported membership of rfc1234 in bcp1 via sync to the rfc-index"], + ["bcp_history_marker", "No history of BCP1 is currently available in the datatracker before this point"], + ["published_rfc", "RFC published"] ] for index, [event_type, desc] in enumerate(expected_events): self.assertEqual(rfc_events[index].type, event_type) if index == 0: - self.assertIn( - "Received changes through RFC Editor sync (created document RFC 1234,", - rfc_events[0].desc, - ) - self.assertIn( - f"created became rfc relationship between {rfc_doc.came_from_draft().name} and RFC 1234", - rfc_events[0].desc, - ) + self.assertIn("Received changes through RFC Editor sync (created document RFC 1234,", rfc_events[0].desc) + self.assertIn(f"created became rfc relationship between {rfc_doc.came_from_draft().name} and RFC 1234", rfc_events[0].desc) self.assertIn("set title to 'A Testing RFC'", rfc_events[0].desc) - self.assertIn( - "set abstract to 'This is some interesting text.'", - rfc_events[0].desc, - ) + self.assertIn("set abstract to 'This is some interesting text.'", rfc_events[0].desc) self.assertIn("set pages to 42", rfc_events[0].desc) - self.assertIn( - "set standardization level to Proposed Standard", rfc_events[0].desc - ) - self.assertIn( - f"added RFC published event at {rfc_events[0].time.astimezone(RPC_TZINFO):%Y-%m-%d}", - rfc_events[0].desc, - ) - self.assertIn( - "created updates relation between RFC 1234 and RFC 123", - rfc_events[0].desc, - ) + self.assertIn("set standardization level to Proposed Standard", rfc_events[0].desc) + self.assertIn(f"added RFC published event at {rfc_events[0].time.astimezone(RPC_TZINFO):%Y-%m-%d}", rfc_events[0].desc) + self.assertIn("created updates relation between RFC 1234 and RFC 123", rfc_events[0].desc) self.assertIn("added Errata tag", rfc_events[0].desc) else: self.assertEqual(rfc_events[index].desc, desc) self.assertEqual(rfc_events[7].time.astimezone(RPC_TZINFO).date(), today) for subseries_name in ["bcp1", "fyi1", "std2"]: - sub = Document.objects.filter( - type_id=subseries_name[:3], name=subseries_name - ).first() + sub = Document.objects.filter(type_id=subseries_name[:3],name=subseries_name).first() self.assertIsNotNone(sub, f"{subseries_name} not created") self.assertTrue(rfc_doc in sub.contains()) self.assertTrue(sub in rfc_doc.part_of()) @@ -558,16 +421,8 @@ def test_rfc_index(self): # self.assertTrue(DocAlias.objects.filter(name="bcp1", docs=rfc_doc)) # self.assertTrue(DocAlias.objects.filter(name="fyi1", docs=rfc_doc)) # self.assertTrue(DocAlias.objects.filter(name="std1", docs=rfc_doc)) - self.assertTrue( - RelatedDocument.objects.filter( - source=rfc_doc, target__name="rfc123", relationship="updates" - ).exists() - ) - self.assertTrue( - RelatedDocument.objects.filter( - source=draft_doc, target=rfc_doc, relationship="became_rfc" - ).exists() - ) + self.assertTrue(RelatedDocument.objects.filter(source=rfc_doc, target__name="rfc123", relationship="updates").exists()) + self.assertTrue(RelatedDocument.objects.filter(source=draft_doc, target=rfc_doc, relationship="became_rfc").exists()) self.assertEqual(rfc_doc.title, "A Testing RFC") self.assertEqual(rfc_doc.abstract, "This is some interesting text.") self.assertEqual(rfc_doc.std_level_id, "ps") @@ -587,16 +442,12 @@ def test_rfc_index(self): self.assertEqual(changes[1]["rfc_published"], True) # make sure we can apply it again with no changes - changed = list( - rfceditor.update_docs_from_rfc_index( - data, errata, today - datetime.timedelta(days=30) - ) - ) + changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30))) self.assertEqual(len(changed), 0) def _generate_rfc_queue_xml(self, draft, state, auth48_url=None): """Generate an RFC queue xml string for a draft""" - t = """ + t = '''
%(name)s-%(rev)s.txt @@ -615,30 +466,24 @@ def _generate_rfc_queue_xml(self, draft, state, auth48_url=None): %(group)s
-""" % dict( - name=draft.name, - rev=draft.rev, - title=draft.title, - group=draft.group.name, - ref="draft-ietf-test", - state=state, - auth48_url=(auth48_url or ""), - ) - t = t.replace("\n", "") # strip empty auth48-url tags +''' % dict(name=draft.name, + rev=draft.rev, + title=draft.title, + group=draft.group.name, + ref="draft-ietf-test", + state=state, + auth48_url=(auth48_url or '')) + t = t.replace('\n', '') # strip empty auth48-url tags return t def test_rfc_queue(self): - draft = WgDraftFactory( - states=[("draft-iesg", "ann")], ad=Person.objects.get(user__username="ad") - ) - draft.action_holders.add( - draft.ad - ) # add an action holder so we can test that it's removed later + draft = WgDraftFactory(states=[('draft-iesg','ann')], ad=Person.objects.get(user__username='ad')) + draft.action_holders.add(draft.ad) # add an action holder so we can test that it's removed later expected_auth48_url = "http://www.rfc-editor.org/auth48/rfc1234" - t = self._generate_rfc_queue_xml( - draft, state="EDIT*R*A(1G)", auth48_url=expected_auth48_url - ) + t = self._generate_rfc_queue_xml(draft, + state='EDIT*R*A(1G)', + auth48_url=expected_auth48_url) drafts, warnings = rfceditor.parse_queue(io.StringIO(t)) # rfceditor.parse_queue() is tested independently; just sanity check here @@ -655,16 +500,11 @@ def test_rfc_queue(self): self.assertEqual(draft.get_state_slug("draft-rfceditor"), "edit") self.assertEqual(draft.get_state_slug("draft-iesg"), "rfcqueue") self.assertCountEqual(draft.action_holders.all(), []) - self.assertEqual( - set(draft.tags.all()), - set(DocTagName.objects.filter(slug__in=("iana", "ref"))), - ) + self.assertEqual(set(draft.tags.all()), set(DocTagName.objects.filter(slug__in=("iana", "ref")))) events = draft.docevent_set.all() - self.assertEqual(events[0].type, "changed_state") # changed draft-iesg state + self.assertEqual(events[0].type, "changed_state") # changed draft-iesg state self.assertEqual(events[1].type, "changed_action_holders") - self.assertEqual( - events[2].type, "changed_state" - ) # changed draft-rfceditor state + self.assertEqual(events[2].type, "changed_state") # changed draft-rfceditor state self.assertEqual(events[3].type, "rfc_editor_received_announcement") self.assertEqual(len(outbox), mailbox_before + 1) @@ -678,31 +518,19 @@ def test_rfc_queue(self): def test_rfceditor_parse_queue(self): """Test that rfceditor.parse_queue() behaves as expected. - Currently does a limited test - old comment was + Currently does a limited test - old comment was "currently, we only check what we actually use". """ - draft = WgDraftFactory(states=[("draft-iesg", "ann")]) - t = self._generate_rfc_queue_xml( - draft, - state="EDIT*R*A(1G)", - auth48_url="http://www.rfc-editor.org/auth48/rfc1234", - ) + draft = WgDraftFactory(states=[('draft-iesg','ann')]) + t = self._generate_rfc_queue_xml(draft, + state='EDIT*R*A(1G)', + auth48_url="http://www.rfc-editor.org/auth48/rfc1234") drafts, warnings = rfceditor.parse_queue(io.StringIO(t)) self.assertEqual(len(drafts), 1) self.assertEqual(len(warnings), 0) - ( - draft_name, - date_received, - state, - tags, - missref_generation, - stream, - auth48, - cluster, - refs, - ) = drafts[0] + draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs = drafts[0] self.assertEqual(draft_name, draft.name) self.assertEqual(state, "EDIT") self.assertEqual(set(tags), set(["iana", "ref"])) @@ -710,66 +538,63 @@ def test_rfceditor_parse_queue(self): def test_rfceditor_parse_queue_TI_state(self): # Test with TI state introduced 11 Sep 2019 - draft = WgDraftFactory(states=[("draft-iesg", "ann")]) - t = self._generate_rfc_queue_xml( - draft, state="TI", auth48_url="http://www.rfc-editor.org/auth48/rfc1234" - ) + draft = WgDraftFactory(states=[('draft-iesg','ann')]) + t = self._generate_rfc_queue_xml(draft, + state='TI', + auth48_url="http://www.rfc-editor.org/auth48/rfc1234") __, warnings = rfceditor.parse_queue(io.StringIO(t)) self.assertEqual(len(warnings), 0) def _generate_rfceditor_update(self, draft, state, tags=None, auth48_url=None): """Helper to generate fake output from rfceditor.parse_queue()""" - return [ - [ - draft.name, # draft_name - "2020-06-03", # date_received - state, - tags or [], - "1", # missref_generation - "ietf", # stream - auth48_url or "", - "", # cluster - ["draft-ietf-test"], # refs - ] - ] + return [[ + draft.name, # draft_name + '2020-06-03', # date_received + state, + tags or [], + '1', # missref_generation + 'ietf', # stream + auth48_url or '', + '', # cluster + ['draft-ietf-test'], # refs + ]] def test_update_draft_auth48_url(self): """Test that auth48 URLs are handled correctly.""" - draft = WgDraftFactory(states=[("draft-iesg", "ann")]) + draft = WgDraftFactory(states=[('draft-iesg','ann')]) # Step 1 setup: update to a state with no auth48 URL changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state="EDIT") + self._generate_rfceditor_update(draft, state='EDIT') ) self.assertEqual(len(changed), 1) self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id="auth48").first() + auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() self.assertIsNone(auth48_docurl) # Step 2: update to auth48 state with auth48 URL changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update( - draft, state="AUTH48", auth48_url="http://www.rfc-editor.org/rfc1234" - ) + self._generate_rfceditor_update(draft, state='AUTH48', auth48_url='http://www.rfc-editor.org/rfc1234') ) self.assertEqual(len(changed), 1) self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id="auth48").first() + auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() self.assertIsNotNone(auth48_docurl) - self.assertEqual(auth48_docurl.url, "http://www.rfc-editor.org/rfc1234") + self.assertEqual(auth48_docurl.url, 'http://www.rfc-editor.org/rfc1234') # Step 3: update to auth48-done state without auth48 URL changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state="AUTH48-DONE") + self._generate_rfceditor_update(draft, state='AUTH48-DONE') ) self.assertEqual(len(changed), 1) self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id="auth48").first() + auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() self.assertIsNone(auth48_docurl) class DiscrepanciesTests(TestCase): def test_discrepancies(self): + # draft approved but no RFC Editor state doc = Document.objects.create(name="draft-ietf-test1", type_id="draft") doc.set_state(State.objects.get(used=True, type="draft-iesg", slug="ann")) @@ -780,9 +605,7 @@ def test_discrepancies(self): # draft with IANA state "In Progress" but RFC Editor state not IANA doc = Document.objects.create(name="draft-ietf-test2", type_id="draft") doc.set_state(State.objects.get(used=True, type="draft-iesg", slug="rfcqueue")) - doc.set_state( - State.objects.get(used=True, type="draft-iana-action", slug="inprog") - ) + doc.set_state(State.objects.get(used=True, type="draft-iana-action", slug="inprog")) doc.set_state(State.objects.get(used=True, type="draft-rfceditor", slug="auth")) r = self.client.get(urlreverse("ietf.sync.views.discrepancies")) @@ -792,9 +615,7 @@ def test_discrepancies(self): # but RFC Editor state is IANA doc = Document.objects.create(name="draft-ietf-test3", type_id="draft") doc.set_state(State.objects.get(used=True, type="draft-iesg", slug="rfcqueue")) - doc.set_state( - State.objects.get(used=True, type="draft-iana-action", slug="waitrfc") - ) + doc.set_state(State.objects.get(used=True, type="draft-iana-action", slug="waitrfc")) doc.set_state(State.objects.get(used=True, type="draft-rfceditor", slug="iana")) r = self.client.get(urlreverse("ietf.sync.views.discrepancies")) @@ -809,30 +630,21 @@ def test_discrepancies(self): r = self.client.get(urlreverse("ietf.sync.views.discrepancies")) self.assertContains(r, doc.name) - class RFCEditorUndoTests(TestCase): def test_rfceditor_undo(self): draft = WgDraftFactory() - e1 = add_state_change_event( - draft, - Person.objects.get(name="(System)"), - None, - State.objects.get(used=True, type="draft-rfceditor", slug="auth"), - ) + e1 = add_state_change_event(draft, Person.objects.get(name="(System)"), None, + State.objects.get(used=True, type="draft-rfceditor", slug="auth")) e1.desc = "First" e1.save() - e2 = add_state_change_event( - draft, - Person.objects.get(name="(System)"), - None, - State.objects.get(used=True, type="draft-rfceditor", slug="edit"), - ) + e2 = add_state_change_event(draft, Person.objects.get(name="(System)"), None, + State.objects.get(used=True, type="draft-rfceditor", slug="edit")) e2.desc = "Second" e2.save() - - url = urlreverse("ietf.sync.views.rfceditor_undo") + + url = urlreverse('ietf.sync.views.rfceditor_undo') login_testing_unauthorized(self, "rfc", url) # get @@ -860,85 +672,3 @@ def test_rfceditor_undo(self): e.content_type.model_class().objects.create(**json.loads(e.json)) self.assertTrue(StateDocEvent.objects.filter(desc="First", doc=draft)) - - -class TaskTests(TestCase): - @override_settings( - RFC_EDITOR_INDEX_URL="https://rfc-editor.example.com/index/", - RFC_EDITOR_ERRATA_JSON_URL="https://rfc-editor.example.com/errata/", - ) - @mock.patch("ietf.sync.tasks.update_docs_from_rfc_index") - @mock.patch("ietf.sync.tasks.parse_index") - @mock.patch("ietf.sync.tasks.requests.get") - def test_rfc_editor_index_update_task( - self, requests_get_mock, parse_index_mock, update_docs_mock - ): - @dataclass - class MockIndexData: - """Mock index item that claims to be a specified length""" - length: int - - def __len__(self): - return self.length - - @dataclass - class MockResponse: - """Mock object that contains text and json() that claims to be a specified length""" - text: str - json_length: int = 0 - - def json(self): - return MockIndexData(length=self.json_length) - - # Response objects - index_response = MockResponse(text="this is the index") - errata_response = MockResponse( - text="these are the errata", json_length=rfceditor.MIN_ERRATA_RESULTS - ) - - # Test with full_index = False - requests_get_mock.side_effect = (index_response, errata_response) # will step through these - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) - update_docs_mock.return_value = [] # not tested - - tasks.rfc_editor_index_update_task(full_index=False) - - # Check parse_index() call - self.assertTrue(parse_index_mock.called) - (parse_index_args, _) = parse_index_mock.call_args - self.assertEqual( - parse_index_args[0].read(), # arg is a StringIO - "this is the index", - "parse_index is called with the index text in a StringIO", - ) - - # Check update_docs_from_rfc_index call - self.assertTrue(update_docs_mock.called) - (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args - self.assertEqual( - update_docs_args, (parse_index_mock.return_value, errata_response.json()) - ) - self.assertIsNotNone(update_docs_kwargs["skip_older_than_date"]) - - # Test again with full_index = True - requests_get_mock.side_effect = (index_response, errata_response) # will step through these - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) - update_docs_mock.return_value = [] # not tested - tasks.rfc_editor_index_update_task(full_index=True) - - # Check parse_index() call - self.assertTrue(parse_index_mock.called) - (parse_index_args, _) = parse_index_mock.call_args - self.assertEqual( - parse_index_args[0].read(), # arg is a StringIO - "this is the index", - "parse_index is called with the index text in a StringIO", - ) - - # Check update_docs_from_rfc_index call - self.assertTrue(update_docs_mock.called) - (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args - self.assertEqual( - update_docs_args, (parse_index_mock.return_value, errata_response.json()) - ) - self.assertIsNone(update_docs_kwargs["skip_older_than_date"]) From 9177c4f07c2267f5fd804d974181a5c1631e3022 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Jan 2024 16:42:31 -0400 Subject: [PATCH 08/11] test: Test rfc_editor_index_update_task This time without reformatting the entire file... --- ietf/sync/tests.py | 92 ++++++++++++++++++++++++++++++++++++++++++++- ietf/utils/tests.py | 5 +++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 6ac8f4afb03..20322600509 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -9,9 +9,12 @@ import mock import quopri +from dataclasses import dataclass + from django.conf import settings from django.urls import reverse as urlreverse from django.utils import timezone +from django.test.utils import override_settings import debug # pyflakes:ignore @@ -20,7 +23,7 @@ from ietf.doc.utils import add_state_change_event from ietf.group.factories import GroupFactory from ietf.person.models import Person -from ietf.sync import iana, rfceditor +from ietf.sync import iana, rfceditor, tasks from ietf.utils.mail import outbox, empty_outbox from ietf.utils.test_utils import login_testing_unauthorized from ietf.utils.test_utils import TestCase @@ -672,3 +675,90 @@ def test_rfceditor_undo(self): e.content_type.model_class().objects.create(**json.loads(e.json)) self.assertTrue(StateDocEvent.objects.filter(desc="First", doc=draft)) + + +class TaskTests(TestCase): + @override_settings( + RFC_EDITOR_INDEX_URL="https://rfc-editor.example.com/index/", + RFC_EDITOR_ERRATA_JSON_URL="https://rfc-editor.example.com/errata/", + ) + @mock.patch("ietf.sync.tasks.update_docs_from_rfc_index") + @mock.patch("ietf.sync.tasks.parse_index") + @mock.patch("ietf.sync.tasks.requests.get") + def test_rfc_editor_index_update_task( + self, requests_get_mock, parse_index_mock, update_docs_mock + ): + """rfc_editor_index_update_task calls helpers correctly + + This tests that data flow is as expected. Assumes the individual helpers are + separately tested to function correctly. + """ + @dataclass + class MockIndexData: + """Mock index item that claims to be a specified length""" + length: int + + def __len__(self): + return self.length + + @dataclass + class MockResponse: + """Mock object that contains text and json() that claims to be a specified length""" + text: str + json_length: int = 0 + + def json(self): + return MockIndexData(length=self.json_length) + + # Response objects + index_response = MockResponse(text="this is the index") + errata_response = MockResponse( + text="these are the errata", json_length=rfceditor.MIN_ERRATA_RESULTS + ) + + # Test with full_index = False + requests_get_mock.side_effect = (index_response, errata_response) # will step through these + parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) + update_docs_mock.return_value = [] # not tested + + tasks.rfc_editor_index_update_task(full_index=False) + + # Check parse_index() call + self.assertTrue(parse_index_mock.called) + (parse_index_args, _) = parse_index_mock.call_args + self.assertEqual( + parse_index_args[0].read(), # arg is a StringIO + "this is the index", + "parse_index is called with the index text in a StringIO", + ) + + # Check update_docs_from_rfc_index call + self.assertTrue(update_docs_mock.called) + (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args + self.assertEqual( + update_docs_args, (parse_index_mock.return_value, errata_response.json()) + ) + self.assertIsNotNone(update_docs_kwargs["skip_older_than_date"]) + + # Test again with full_index = True + requests_get_mock.side_effect = (index_response, errata_response) # will step through these + parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) + update_docs_mock.return_value = [] # not tested + tasks.rfc_editor_index_update_task(full_index=True) + + # Check parse_index() call + self.assertTrue(parse_index_mock.called) + (parse_index_args, _) = parse_index_mock.call_args + self.assertEqual( + parse_index_args[0].read(), # arg is a StringIO + "this is the index", + "parse_index is called with the index text in a StringIO", + ) + + # Check update_docs_from_rfc_index call + self.assertTrue(update_docs_mock.called) + (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args + self.assertEqual( + update_docs_args, (parse_index_mock.return_value, errata_response.json()) + ) + self.assertIsNone(update_docs_kwargs["skip_older_than_date"]) diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 499b874886c..03b6c8cceb3 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -598,3 +598,8 @@ class TestForm(Form): self.assertTrue(changed_form.has_changed()) unchanged_form = TestForm(initial={'test_field': [1]}, data={'test_field': [1]}) self.assertFalse(unchanged_form.has_changed()) + + +class TaskTests(TestCase): + def test_send_scheduled_mail_task(self): + SendQueue From c18805adc0a2e7da2c8f6ce96080ca46d1eb7756 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Jan 2024 16:54:02 -0400 Subject: [PATCH 09/11] chore: Remove accidentally committed fragment --- ietf/utils/tests.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 03b6c8cceb3..499b874886c 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -598,8 +598,3 @@ class TestForm(Form): self.assertTrue(changed_form.has_changed()) unchanged_form = TestForm(initial={'test_field': [1]}, data={'test_field': [1]}) self.assertFalse(unchanged_form.has_changed()) - - -class TaskTests(TestCase): - def test_send_scheduled_mail_task(self): - SendQueue From 953ee5cbadb83e82390fd19e3e9c0d8ed4895523 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Jan 2024 17:17:48 -0400 Subject: [PATCH 10/11] test: Annotate function to satisfy mypy --- ietf/sync/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 20322600509..d660774d07a 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -687,7 +687,7 @@ class TaskTests(TestCase): @mock.patch("ietf.sync.tasks.requests.get") def test_rfc_editor_index_update_task( self, requests_get_mock, parse_index_mock, update_docs_mock - ): + ) -> None: # the annotation here prevents mypy from complaining about annotation-unchecked """rfc_editor_index_update_task calls helpers correctly This tests that data flow is as expected. Assumes the individual helpers are From 05516765b501bd5f1e6b286e3f4f9a114defdb66 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Jan 2024 17:23:21 -0400 Subject: [PATCH 11/11] chore: Remove unused imports --- ietf/utils/tasks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ietf/utils/tasks.py b/ietf/utils/tasks.py index e889f08720f..efd776b9d87 100644 --- a/ietf/utils/tasks.py +++ b/ietf/utils/tasks.py @@ -7,9 +7,6 @@ from ietf.message.utils import send_scheduled_message_from_send_queue from ietf.message.models import SendQueue -from ietf.review.tasks import send_review_reminders_task -from ietf.stats.tasks import fetch_meeting_attendance_task -from ietf.sync.tasks import rfc_editor_index_update_task from ietf.utils import log from ietf.utils.mail import log_smtp_exception, send_error_email