From 042c6b4ce521d30cb8a225c132ab0515e03bb1eb Mon Sep 17 00:00:00 2001 From: russhousley Date: Sat, 16 Mar 2024 03:47:05 -0400 Subject: [PATCH 1/6] Add celery tasks for idnits2 and bibxml file generation --- ietf/doc/tasks.py | 50 +++++++++++++++++++++++++++ ietf/doc/tests_tasks.py | 75 +++++++++++++++++++++++++++++++++++++++-- ietf/doc/utils.py | 42 ++++++++++++++++++++++- 3 files changed, 164 insertions(+), 3 deletions(-) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index a2e83e9e269..59e9226224b 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -2,13 +2,20 @@ # # Celery task definitions # +import os import datetime import debug # pyflakes:ignore +from django.conf import settings + from celery import shared_task from ietf.utils import log from ietf.utils.timezone import datetime_today +from ietf.doc.utils import generate_idnits2_rfcs_obsoleted +from ietf.doc.utils import generate_idnits2_rfc_status +from ietf.doc.utils import bibxml_for_all_drafts +from ietf.doc.utils import bibxml_for_recent_drafts from .expire import ( in_draft_expire_freeze, @@ -54,3 +61,46 @@ def expire_ids_task(): def notify_expirations_task(notify_days=14): for doc in get_soon_to_expire_drafts(notify_days): send_expire_warning_for_draft(doc) + + +@shared_task +def generate_idnits2_rfcs_obsoleted_task(): + filename = os.path.join(settings.DERIVED_DIR, 'idnits2-rfcs-obsoleted') + blob = generate_idnits2_rfcs_obsoleted() + try: + f = open(filename, 'wb') + f.write(blob.encode('utf-8')) + except Exception as e: + log.log('failed to write idnits2-rfcs-obsoleted: ' + str(e)) + raise e + + +@shared_task +def generate_idnits2_rfc_status_task(): + filename = os.path.join(settings.DERIVED_DIR, 'idnits2-rfc-status') + blob = generate_idnits2_rfc_status() + try: + f = open(filename, 'wb') + f.write(blob.encode('utf-8')) + except Exception as e: + log.log('failed to write idnits2-rfc-status: ' + str(e)) + raise e + + +@shared_task +def generate_bibxml_files_for_all_drafts_task(): + bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') + if not os.path.exists(bibxmldir): + log.log('%s directory needs to be created' % bibxmldir) + print(bibxmldir) + raise FileNotFoundError(bibxmldir) + bibxml_for_all_drafts(bibxmldir) + + +@shared_task +def generate_bibxml_files_for_recent_drafts_task(days=7): + bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') + if not os.path.exists(bibxmldir): + log.log('%s directory needs to be created' % bibxmldir) + raise FileNotFoundError(bibxmldir) + bibxml_for_recent_drafts(bibxmldir, days=days) diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 931ed438dc0..5fbdc207183 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -1,13 +1,21 @@ # Copyright The IETF Trust 2024, All Rights Reserved + +import io +import os import mock +from django.urls import reverse as urlreverse +from django.conf import settings + from ietf.utils.test_utils import TestCase from ietf.utils.timezone import datetime_today +from ietf.utils.test_utils import unicontent -from .factories import DocumentFactory +from .factories import DocumentFactory, IndividualDraftFactory, WgRfcFactory from .models import Document from .tasks import expire_ids_task, notify_expirations_task - +from .tasks import generate_idnits2_rfcs_obsoleted_task, generate_idnits2_rfc_status_task +from .tasks import generate_bibxml_files_for_all_drafts_task, generate_bibxml_files_for_recent_drafts_task class TaskTests(TestCase): @@ -61,3 +69,66 @@ def test_notify_expirations_task(self, get_drafts_mock, send_warning_mock): notify_expirations_task() self.assertEqual(send_warning_mock.call_count, 1) self.assertEqual(send_warning_mock.call_args[0], ("sentinel",)) + + +class Idnits2SupportTests(TestCase): + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR'] + + def test_generate_idnits2_rfcs_obsoleted_task(self): + rfc = WgRfcFactory(rfc_number=1001) + WgRfcFactory(rfc_number=1003,relations=[('obs',rfc)]) + rfc = WgRfcFactory(rfc_number=1005) + WgRfcFactory(rfc_number=1007,relations=[('obs',rfc)]) + url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted') + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + generate_idnits2_rfcs_obsoleted_task() + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.content, b'1001 1003\n1005 1007\n') + + def test_generate_idnits2_rfc_status_task(self): + for slug in ('bcp', 'ds', 'exp', 'hist', 'inf', 'std', 'ps', 'unkn'): + WgRfcFactory(std_level_id=slug) + url = urlreverse('ietf.doc.views_doc.idnits2_rfc_status') + r = self.client.get(url) + self.assertEqual(r.status_code,404) + generate_idnits2_rfc_status_task() + r = self.client.get(url) + self.assertEqual(r.status_code,200) + blob = unicontent(r).replace('\n','') + self.assertEqual(blob[6312-1],'O') + + +class BIBXMLSupportTests(TestCase): + def test_generate_bibxml_files_for_all_drafts_task(self): + draft = IndividualDraftFactory.create() + filename = 'reference.I-D.%s-%s.xml' % (draft.name, draft.rev) + bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') + os.mkdir(bibxmldir) + filepath = os.path.join(bibxmldir, filename) + self.assertFalse(os.path.exists(filepath)) + generate_bibxml_files_for_all_drafts_task() + self.assertTrue(os.path.exists(filepath)) + with io.open(filepath, encoding='utf-8') as f: + content = f.read() + self.assertIn(draft.title, content) + + def test_generate_bibxml_files_for_recent_drafts_task(self): + draft = IndividualDraftFactory.create() + filename = 'reference.I-D.%s-%s.xml' % (draft.name, draft.rev) + bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') + os.mkdir(bibxmldir) + filepath = os.path.join(bibxmldir, filename) + self.assertFalse(os.path.exists(filepath)) + generate_bibxml_files_for_recent_drafts_task(days=7) + self.assertTrue(os.path.exists(filepath)) + with io.open(filepath, encoding='utf-8') as f: + content = f.read() + self.assertIn(draft.title, content) + + def test_generate_bibxml_files_for_recent_drafts_task_with_bad_vakue(self): + bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') + os.mkdir(bibxmldir) + with self.assertRaises(ValueError): + generate_bibxml_files_for_recent_drafts_task(days=0) diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index ad1c2af2233..5a62d7769c6 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2011-2020, All Rights Reserved +# Copyright The IETF Trust 2011-2024, All Rights Reserved # -*- coding: utf-8 -*- @@ -1224,6 +1224,19 @@ def fuzzy_find_documents(name, rev=None): FoundDocuments = namedtuple('FoundDocuments', 'documents matched_name matched_rev') return FoundDocuments(docs, name, rev) +def write_bibxml_file_if_changed(filepath, content): + # normalize new content + content = re.sub(r'\r\n?', r'\n', content) + try: + with io.open(filepath, encoding='utf-8') as f: + old = f.read() + except IOError: + old = "" + if old.strip() != content.strip(): + with io.open(filepath, "w", encoding='utf-8') as f: + f.write(content) + + def bibxml_for_draft(doc, rev=None): if rev is not None and rev != doc.rev: @@ -1259,6 +1272,33 @@ def bibxml_for_draft(doc, rev=None): return render_to_string('doc/bibxml.xml', {'name':name, 'doc':doc, 'doc_bibtype':'I-D', 'settings':settings}) +def bibxml_for_all_drafts(bibxmldir): + doc_events = NewRevisionDocEvent.objects.filter(type='new_revision', doc__type_id='draft').order_by('time') + for e in doc_events: + try: + bibxml = bibxml_for_draft(e.doc, e.rev) + ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (e.doc.name, e.rev)) + write_bibxml_file_if_changed(ref_rev_file_name, bibxml) + except Exception as ee: + log.log('\n%s-%s: %s\n' % (e.doc.name, e.doc.rev, str(ee))) + raise ee + + +def bibxml_for_recent_drafts(bibxmldir, days=7): + if days < 1: + raise ValueError() + start = timezone.now() - datetime.timedelta(days=days) + doc_events = NewRevisionDocEvent.objects.filter(type='new_revision', doc__type_id='draft', time__gte=start).order_by('time') + for e in doc_events: + try: + bibxml = bibxml_for_draft(e.doc, e.rev) + ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (e.doc.name, e.rev)) + write_bibxml_file_if_changed(ref_rev_file_name, bibxml) + except Exception as ee: + log.log('\n%s-%s: %s\n' % (e.doc.name, e.doc.rev, str(ee))) + raise ee + + class DraftAliasGenerator: days = 2 * 365 From 6e617ac7cdd4c2bff6d523e327f9926bdab52cef Mon Sep 17 00:00:00 2001 From: Russ Housley Date: Wed, 20 Mar 2024 20:47:48 -0400 Subject: [PATCH 2/6] Update tests_tasks.py Fix typo: bad_vakue -> bad_value --- ietf/doc/tests_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 5fbdc207183..5fdf3b13710 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -127,7 +127,7 @@ def test_generate_bibxml_files_for_recent_drafts_task(self): content = f.read() self.assertIn(draft.title, content) - def test_generate_bibxml_files_for_recent_drafts_task_with_bad_vakue(self): + def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value(self): bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') os.mkdir(bibxmldir) with self.assertRaises(ValueError): From 01753af09bf7285fa4ad68c20c461cf2cce9756c Mon Sep 17 00:00:00 2001 From: Russ Housley Date: Wed, 20 Mar 2024 20:50:05 -0400 Subject: [PATCH 3/6] Update utils.py Don't raise error inbibxml_for_all_drafts --- ietf/doc/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 5a62d7769c6..823a891281f 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -1281,7 +1281,6 @@ def bibxml_for_all_drafts(bibxmldir): write_bibxml_file_if_changed(ref_rev_file_name, bibxml) except Exception as ee: log.log('\n%s-%s: %s\n' % (e.doc.name, e.doc.rev, str(ee))) - raise ee def bibxml_for_recent_drafts(bibxmldir, days=7): From 986fe0929aec9b8a2daa0406f0344b283a90e6f2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 15 Jul 2024 14:06:46 -0300 Subject: [PATCH 4/6] chore: fixup merge --- ietf/doc/tasks.py | 2 + ietf/doc/tests_tasks.py | 175 ++++++++++++++-------------------------- 2 files changed, 64 insertions(+), 113 deletions(-) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index 209db035a44..f1de459dd84 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -105,6 +105,8 @@ def generate_draft_bibxml_files_task(days=7, process_all=False): If process_all is False (the default), processes only docs with new revisions in the last specified number of days. """ + if not process_all and days < 1: + raise ValueError("Must call with days >= 1 or process_all=True") ensure_draft_bibxml_path_exists() doc_events = NewRevisionDocEvent.objects.filter( type="new_revision", diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index de09810b31e..cbb6e1a1e86 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -1,20 +1,17 @@ # Copyright The IETF Trust 2024, All Rights Reserved import datetime -import io import mock import os from pathlib import Path from django.conf import settings -from django.urls import reverse as urlreverse from django.utils import timezone from ietf.utils.test_utils import TestCase from ietf.utils.timezone import datetime_today -from ietf.utils.test_utils import unicontent -from .factories import DocumentFactory, IndividualDraftFactory, NewRevisionDocEventFactory, WgRfcFactory +from .factories import DocumentFactory, NewRevisionDocEventFactory from .models import Document, NewRevisionDocEvent from .tasks import ( expire_ids_task, @@ -26,8 +23,6 @@ ) class TaskTests(TestCase): - settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ["DERIVED_DIR"] - @mock.patch("ietf.doc.tasks.in_draft_expire_freeze") @mock.patch("ietf.doc.tasks.get_expired_drafts") @mock.patch("ietf.doc.tasks.expirable_drafts") @@ -102,37 +97,42 @@ def test_expire_last_calls_task(self, mock_get_expired, mock_expire): self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1])) self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2])) - @mock.patch("ietf.doc.tasks.generate_idnits2_rfc_status") - def test_generate_idnits2_rfc_status_task(self, mock_generate): + +class Idnits2SupportTests(TestCase): + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR'] + + @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted") + def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate): mock_generate.return_value = "dåtå" - generate_idnits2_rfc_status_task() + generate_idnits2_rfcs_obsoleted_task() self.assertEqual(mock_generate.call_count, 1) self.assertEqual( "dåtå".encode("utf8"), - (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").read_bytes(), + (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(), ) - - @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted") - def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate): + + @mock.patch("ietf.doc.tasks.generate_idnits2_rfc_status") + def test_generate_idnits2_rfc_status_task(self, mock_generate): mock_generate.return_value = "dåtå" - generate_idnits2_rfcs_obsoleted_task() + generate_idnits2_rfc_status_task() self.assertEqual(mock_generate.call_count, 1) self.assertEqual( "dåtå".encode("utf8"), - (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(), + (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").read_bytes(), ) - @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") - @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_draft_bibxml_files_task(self, mock_create, mock_ensure_path): + +class BIBXMLSupportTests(TestCase): + def setUp(self): + super().setUp() now = timezone.now() - very_old_event = NewRevisionDocEventFactory( + self.very_old_event = NewRevisionDocEventFactory( time=now - datetime.timedelta(days=1000), rev="17" ) - old_event = NewRevisionDocEventFactory( + self.old_event = NewRevisionDocEventFactory( time=now - datetime.timedelta(days=8), rev="03" ) - young_event = NewRevisionDocEventFactory( + self.young_event = NewRevisionDocEventFactory( time=now - datetime.timedelta(days=6), rev="06" ) # a couple that should always be ignored @@ -145,125 +145,74 @@ def test_generate_draft_bibxml_files_task(self, mock_create, mock_ensure_path): rev="09", doc__type_id="rfc", ) - # Get rid of the "00" events created by the factories -- they're just noise for this test NewRevisionDocEvent.objects.filter(rev="00").delete() - - # default args - look back 7 days - generate_draft_bibxml_files_task() - self.assertTrue(mock_ensure_path.called) - self.assertCountEqual( - mock_create.call_args_list, [mock.call(young_event.doc, young_event.rev)] - ) - mock_create.reset_mock() - mock_ensure_path.reset_mock() - - # shorter lookback - generate_draft_bibxml_files_task(days=5) - self.assertTrue(mock_ensure_path.called) - self.assertCountEqual(mock_create.call_args_list, []) - mock_create.reset_mock() - mock_ensure_path.reset_mock() - - # longer lookback - generate_draft_bibxml_files_task(days=9) + + @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") + @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") + def test_generate_bibxml_files_for_all_drafts_task(self, mock_create, mock_ensure_path): + generate_draft_bibxml_files_task(process_all=True) self.assertTrue(mock_ensure_path.called) self.assertCountEqual( mock_create.call_args_list, [ - mock.call(young_event.doc, young_event.rev), - mock.call(old_event.doc, old_event.rev), + mock.call(self.young_event.doc, self.young_event.rev), + mock.call(self.old_event.doc, self.old_event.rev), + mock.call(self.very_old_event.doc, self.very_old_event.rev), ], ) mock_create.reset_mock() mock_ensure_path.reset_mock() - - # everything + + # everything should still be tried, even if there's an exception + mock_create.side_effect = RuntimeError generate_draft_bibxml_files_task(process_all=True) self.assertTrue(mock_ensure_path.called) self.assertCountEqual( mock_create.call_args_list, [ - mock.call(young_event.doc, young_event.rev), - mock.call(old_event.doc, old_event.rev), - mock.call(very_old_event.doc, very_old_event.rev), + mock.call(self.young_event.doc, self.young_event.rev), + mock.call(self.old_event.doc, self.old_event.rev), + mock.call(self.very_old_event.doc, self.very_old_event.rev), ], ) + + @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") + @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") + def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_ensure_path): + # default args - look back 7 days + generate_draft_bibxml_files_task() + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual( + mock_create.call_args_list, [mock.call(self.young_event.doc, self.young_event.rev)] + ) mock_create.reset_mock() mock_ensure_path.reset_mock() - - # everything should still be tried, even if there's an exception - mock_create.side_effect = RuntimeError - generate_draft_bibxml_files_task(process_all=True) + + # shorter lookback + generate_draft_bibxml_files_task(days=5) + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual(mock_create.call_args_list, []) + mock_create.reset_mock() + mock_ensure_path.reset_mock() + + # longer lookback + generate_draft_bibxml_files_task(days=9) self.assertTrue(mock_ensure_path.called) self.assertCountEqual( mock_create.call_args_list, [ - mock.call(young_event.doc, young_event.rev), - mock.call(old_event.doc, old_event.rev), - mock.call(very_old_event.doc, very_old_event.rev), + mock.call(self.young_event.doc, self.young_event.rev), + mock.call(self.old_event.doc, self.old_event.rev), ], ) - -class Idnits2SupportTests(TestCase): - settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR'] - - def test_generate_idnits2_rfcs_obsoleted_task(self): - rfc = WgRfcFactory(rfc_number=1001) - WgRfcFactory(rfc_number=1003,relations=[('obs',rfc)]) - rfc = WgRfcFactory(rfc_number=1005) - WgRfcFactory(rfc_number=1007,relations=[('obs',rfc)]) - url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted') - r = self.client.get(url) - self.assertEqual(r.status_code, 404) - generate_idnits2_rfcs_obsoleted_task() - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - self.assertEqual(r.content, b'1001 1003\n1005 1007\n') - - def test_generate_idnits2_rfc_status_task(self): - for slug in ('bcp', 'ds', 'exp', 'hist', 'inf', 'std', 'ps', 'unkn'): - WgRfcFactory(std_level_id=slug) - url = urlreverse('ietf.doc.views_doc.idnits2_rfc_status') - r = self.client.get(url) - self.assertEqual(r.status_code,404) - generate_idnits2_rfc_status_task() - r = self.client.get(url) - self.assertEqual(r.status_code,200) - blob = unicontent(r).replace('\n','') - self.assertEqual(blob[6312-1],'O') - - -class BIBXMLSupportTests(TestCase): - def test_generate_bibxml_files_for_all_drafts_task(self): - draft = IndividualDraftFactory.create() - filename = 'reference.I-D.%s-%s.xml' % (draft.name, draft.rev) - bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') - os.mkdir(bibxmldir) - filepath = os.path.join(bibxmldir, filename) - self.assertFalse(os.path.exists(filepath)) - generate_draft_bibxml_files_task(process_all=True) - self.assertTrue(os.path.exists(filepath)) - with io.open(filepath, encoding='utf-8') as f: - content = f.read() - self.assertIn(draft.title, content) - - def test_generate_bibxml_files_for_recent_drafts_task(self): - draft = IndividualDraftFactory.create() - filename = 'reference.I-D.%s-%s.xml' % (draft.name, draft.rev) - bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') - os.mkdir(bibxmldir) - filepath = os.path.join(bibxmldir, filename) - self.assertFalse(os.path.exists(filepath)) - generate_draft_bibxml_files_task(days=7) - self.assertTrue(os.path.exists(filepath)) - with io.open(filepath, encoding='utf-8') as f: - content = f.read() - self.assertIn(draft.title, content) - - def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value(self): + @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") + @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") + def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value(self, mock_create, mock_ensure_path): bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') os.mkdir(bibxmldir) with self.assertRaises(ValueError): generate_draft_bibxml_files_task(days=0) + self.assertFalse(mock_create.called) + self.assertFalse(mock_ensure_path.called) From 2f6b986d563995fc75c8d4ae36c65ef9c9127226 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 15 Jul 2024 14:13:09 -0300 Subject: [PATCH 5/6] chore: more merge cleanup --- ietf/doc/utils.py | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 102f45d4b29..a98b46cb509 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -1228,18 +1228,6 @@ def fuzzy_find_documents(name, rev=None): FoundDocuments = namedtuple('FoundDocuments', 'documents matched_name matched_rev') return FoundDocuments(docs, name, rev) -def write_bibxml_file_if_changed(filepath, content): - # normalize new content - content = re.sub(r'\r\n?', r'\n', content) - try: - with io.open(filepath, encoding='utf-8') as f: - old = f.read() - except IOError: - old = "" - if old.strip() != content.strip(): - with io.open(filepath, "w", encoding='utf-8') as f: - f.write(content) - def bibxml_for_draft(doc, rev=None): @@ -1276,32 +1264,6 @@ def bibxml_for_draft(doc, rev=None): return render_to_string('doc/bibxml.xml', {'name':name, 'doc':doc, 'doc_bibtype':'I-D', 'settings':settings}) -def bibxml_for_all_drafts(bibxmldir): - doc_events = NewRevisionDocEvent.objects.filter(type='new_revision', doc__type_id='draft').order_by('time') - for e in doc_events: - try: - bibxml = bibxml_for_draft(e.doc, e.rev) - ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (e.doc.name, e.rev)) - write_bibxml_file_if_changed(ref_rev_file_name, bibxml) - except Exception as ee: - log.log('\n%s-%s: %s\n' % (e.doc.name, e.doc.rev, str(ee))) - - -def bibxml_for_recent_drafts(bibxmldir, days=7): - if days < 1: - raise ValueError() - start = timezone.now() - datetime.timedelta(days=days) - doc_events = NewRevisionDocEvent.objects.filter(type='new_revision', doc__type_id='draft', time__gte=start).order_by('time') - for e in doc_events: - try: - bibxml = bibxml_for_draft(e.doc, e.rev) - ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (e.doc.name, e.rev)) - write_bibxml_file_if_changed(ref_rev_file_name, bibxml) - except Exception as ee: - log.log('\n%s-%s: %s\n' % (e.doc.name, e.doc.rev, str(ee))) - raise ee - - class DraftAliasGenerator: days = 2 * 365 From 02c6b33999dd1066bc7b93a8c3742060d2cf315f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 15 Jul 2024 14:17:07 -0300 Subject: [PATCH 6/6] chore: one last merge cleanup --- ietf/doc/tests_tasks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index cbb6e1a1e86..b75f58656be 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -1,7 +1,6 @@ # Copyright The IETF Trust 2024, All Rights Reserved import datetime import mock -import os from pathlib import Path @@ -210,8 +209,6 @@ def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_en @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value(self, mock_create, mock_ensure_path): - bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') - os.mkdir(bibxmldir) with self.assertRaises(ValueError): generate_draft_bibxml_files_task(days=0) self.assertFalse(mock_create.called)