diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 1e96118e58..25982d76bf 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -32,7 +32,9 @@ EmailPersonSerializer, RfcWithAuthorsSerializer, DraftWithAuthorsSerializer, - NotificationAckSerializer, RfcPubSerializer, RfcFileSerializer, + NotificationAckSerializer, + RfcPubSerializer, + RfcFileSerializer, EditableRfcSerializer, ) from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent @@ -344,9 +346,10 @@ def post(self, request): class RfcAuthorViewSet(viewsets.ReadOnlyModelViewSet): """ViewSet for RfcAuthor model - + Router needs to provide rfc_number as a kwarg """ + api_key_endpoint = "ietf.api.views_rpc" queryset = RfcAuthor.objects.all() @@ -407,7 +410,7 @@ class RfcPubFilesView(APIView): def _fs_destination(self, filename: str | Path) -> Path: """Destination for an uploaded RFC file in the filesystem - + Strips any path components in filename and returns an absolute Path. """ rfc_path = Path(settings.RFC_PATH) @@ -419,7 +422,7 @@ def _fs_destination(self, filename: str | Path) -> Path: def _blob_destination(self, filename: str | Path) -> str: """Destination name for an uploaded RFC file in the blob store - + Strips any path components in filename and returns an absolute Path. """ filename = Path(filename) # could potentially have directory components @@ -472,9 +475,7 @@ def post(self, request): code="files-exist", ) for possible_existing_blob in possible_rfc_blobs: - if exists_in_storage( - kind=blob_kind, name=possible_existing_blob - ): + if exists_in_storage(kind=blob_kind, name=possible_existing_blob): raise Conflict( "Blob(s) already exist for this RFC", code="blobs-exist", @@ -523,7 +524,9 @@ def post(self, request): # Trigger red precomputer needs_updating = [rfc.rfc_number] - for rel in rfc.relateddocument_set.filter(relationship_id__in=["obs","updates"]): + for rel in rfc.relateddocument_set.filter( + relationship_id__in=["obs", "updates"] + ): needs_updating.append(rel.target.rfc_number) trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating)) # Trigger search index update @@ -540,7 +543,7 @@ class RfcIndexView(APIView): @extend_schema( operation_id="refresh_rfc_index", summary="Refresh rfc-index files", - description="Requests creation of rfc-index.xml and rfc-index.txt files", + description="Requests creation of various index files.", responses={202: None}, request=None, ) diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index 4c84dc581e..c48368cccd 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -18,7 +18,13 @@ from ietf.sync import iana from ietf.sync import rfceditor from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue -from ietf.sync.rfcindex import create_rfc_txt_index, create_rfc_xml_index +from ietf.sync.rfcindex import ( + create_bcp_txt_index, + create_fyi_txt_index, + create_rfc_txt_index, + create_rfc_xml_index, + create_std_txt_index, +) from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper from ietf.utils import log from ietf.utils.timezone import date_today @@ -27,13 +33,13 @@ @shared_task def rfc_editor_index_update_task(full_index=False): """Update metadata from the RFC index - + Default is to examine only changes in the past 365 days. Call with full_index=True to update the full RFC index. - + According to comments on the original script, a year's worth took about 20s on production as of August 2022 - + The original rfc-editor-index-update script had a long-disabled provision for running the rebuild_reference_relations scripts after the update. That has not been brought over at all because it should be implemented as its own task if it is needed. @@ -51,7 +57,7 @@ def rfc_editor_index_update_task(full_index=False): timeout=30, # seconds ) except requests.Timeout as exc: - log.log(f'GET request timed out retrieving RFC editor index: {exc}') + log.log(f"GET request timed out retrieving RFC editor index: {exc}") return # failed rfc_index_xml = response.text index_data = rfceditor.parse_index(io.StringIO(rfc_index_xml)) @@ -61,9 +67,9 @@ def rfc_editor_index_update_task(full_index=False): timeout=30, # seconds ) except requests.Timeout as exc: - log.log(f'GET request timed out retrieving RFC editor errata: {exc}') + log.log(f"GET request timed out retrieving RFC editor errata: {exc}") return # failed - errata_data = response.json() + errata_data = response.json() if len(index_data) < rfceditor.MIN_INDEX_RESULTS: log.log("Not enough index entries, only %s" % len(index_data)) return # failed @@ -96,15 +102,15 @@ def rfc_editor_queue_updates_task(): drafts, warnings = parse_queue(io.StringIO(response.text)) for w in warnings: log.log(f"Warning: {w}") - + if len(drafts) < MIN_QUEUE_RESULTS: log.log("Not enough results, only %s" % len(drafts)) return # failed - + changed, warnings = update_drafts_from_queue(drafts) for w in warnings: log.log(f"Warning: {w}") - + for c in changed: log.log(f"Updated {c}") @@ -120,9 +126,11 @@ def iana_changes_update_task(): MAX_INTERVAL_ACCEPTED_BY_IANA = datetime.timedelta(hours=23) start = ( - timezone.now() - - datetime.timedelta(hours=23) - + datetime.timedelta(seconds=CLOCK_SKEW_COMPENSATION,) + timezone.now() + - datetime.timedelta(hours=23) + + datetime.timedelta( + seconds=CLOCK_SKEW_COMPENSATION, + ) ) end = start + datetime.timedelta(hours=23) @@ -133,7 +141,9 @@ def iana_changes_update_task(): # requests if necessary text = iana.fetch_changes_json( - settings.IANA_SYNC_CHANGES_URL, t, min(end, t + MAX_INTERVAL_ACCEPTED_BY_IANA) + settings.IANA_SYNC_CHANGES_URL, + t, + min(end, t + MAX_INTERVAL_ACCEPTED_BY_IANA), ) log.log(f"Retrieved the JSON: {text}") @@ -159,9 +169,9 @@ def iana_protocols_update_task(): # "this needs to be the date where this tool is first deployed" in the original # iana-protocols-updates script)" rfc_must_published_later_than = datetime.datetime( - 2012, - 11, - 26, + 2012, + 11, + 26, tzinfo=datetime.UTC, ) @@ -171,17 +181,17 @@ def iana_protocols_update_task(): timeout=30, ) except requests.Timeout as exc: - log.log(f'GET request timed out retrieving IANA protocols page: {exc}') + log.log(f"GET request timed out retrieving IANA protocols page: {exc}") return rfc_numbers = iana.parse_protocol_page(response.text) def batched(l, n): """Split list l up in batches of max size n. - + For Python 3.12 or later, replace this with itertools.batched() """ - return (l[i:i + n] for i in range(0, len(l), n)) + return (l[i : i + n] for i in range(0, len(l), n)) for batch in batched(rfc_numbers, 100): updated = iana.update_rfc_log_from_protocol_page( @@ -192,6 +202,7 @@ def batched(l, n): for d in updated: log.log("Added history entry for %s" % d.display_name()) + @shared_task def fix_subseries_docevents_task(): """Repairs DocEvents related to bugs around removing docs from subseries @@ -233,6 +244,7 @@ def fix_subseries_docevents_task(): time=obsoleting_time ) + @shared_task def rsync_rfcs_from_rfceditor_task(rfc_numbers: list[int]): log.log(f"Rsyncing rfcs from rfc-editor: {rfc_numbers}") @@ -277,6 +289,32 @@ def load_rfcs_into_blobdb_task(start: int, end: int): @shared_task def create_rfc_index_task(): - create_rfc_txt_index() - create_rfc_xml_index() + try: + create_rfc_txt_index() + except Exception as e: + log.log(f"Error: failure in creating rfc-index.txt. {e}") + pass + try: + create_rfc_xml_index() + except Exception as e: + log.log(f"Error: failure in creating rfc-index.xml. {e}") + pass + + try: + create_bcp_txt_index() + except Exception as e: + log.log(f"Error: failure in creating bcp-index.txt. {e}") + pass + + try: + create_std_txt_index() + except Exception as e: + log.log(f"Error: failure in creating std-index.txt. {e}") + pass + + try: + create_fyi_txt_index() + except Exception as e: + log.log(f"Error: failure in creating fyi-index.txt. {e}") + pass