diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 0db67e126f..180221cffc 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -1,4 +1,5 @@ # Copyright The IETF Trust 2025, All Rights Reserved +import datetime from io import StringIO from pathlib import Path from tempfile import TemporaryDirectory @@ -10,12 +11,15 @@ from django.test.utils import override_settings from django.urls import reverse as urlreverse import mock +from django.utils import timezone from ietf.blobdb.models import Blob from ietf.doc.factories import IndividualDraftFactory, RfcFactory, WgDraftFactory, WgRfcFactory from ietf.doc.models import RelatedDocument, Document from ietf.group.factories import RoleFactory, GroupFactory from ietf.person.factories import PersonFactory +from ietf.sync.rfcindex import rfcindex_is_dirty +from ietf.utils.models import DirtyBits from ietf.utils.test_utils import APITestCase, reload_db_objects @@ -408,8 +412,13 @@ def _valid_post_data(): ) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) - @mock.patch("ietf.api.views_rpc.create_rfc_index_task") - def test_refresh_rfc_index(self, mock_task): + def test_refresh_rfc_index(self): + DirtyBits.objects.create( + slug=DirtyBits.Slugs.RFCINDEX, + dirty_time=timezone.now() - datetime.timedelta(days=1), + processed_time=timezone.now() - datetime.timedelta(hours=12), + ) + self.assertFalse(rfcindex_is_dirty()) url = urlreverse("ietf.api.purple_api.refresh_rfc_index") response = self.client.get(url) self.assertEqual(response.status_code, 403) @@ -417,7 +426,7 @@ def test_refresh_rfc_index(self, mock_task): self.assertEqual(response.status_code, 403) response = self.client.get(url, headers={"X-Api-Key": "valid-token"}) self.assertEqual(response.status_code, 405) - self.assertFalse(mock_task.delay.called) + self.assertFalse(rfcindex_is_dirty()) response = self.client.post(url, headers={"X-Api-Key": "valid-token"}) self.assertEqual(response.status_code, 202) - self.assertTrue(mock_task.delay.called) + self.assertTrue(rfcindex_is_dirty()) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 25982d76bf..6bc45fe3da 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -47,7 +47,7 @@ update_rfc_searchindex_task, ) from ietf.person.models import Email, Person -from ietf.sync.tasks import create_rfc_index_task +from ietf.sync.rfcindex import mark_rfcindex_as_dirty class Conflict(APIException): @@ -548,5 +548,5 @@ class RfcIndexView(APIView): request=None, ) def post(self, request): - create_rfc_index_task.delay() + mark_rfcindex_as_dirty() return Response(status=202) diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index 0ea6fb939f..6864617874 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -1,4 +1,5 @@ # Copyright The IETF Trust 2026, All Rights Reserved +import datetime import json from collections import defaultdict from collections.abc import Container @@ -11,6 +12,7 @@ from django.conf import settings from django.core.files.base import ContentFile +from django.db.models import Q from lxml import etree from django.core.files.storage import storages @@ -22,6 +24,7 @@ from ietf.doc.models import Document from ietf.name.models import StdLevelName from ietf.utils.log import log +from ietf.utils.models import DirtyBits FORMATS_FOR_INDEX = ["txt", "html", "pdf", "xml", "ps"] SS_TXT_MARGIN = 3 @@ -739,3 +742,50 @@ def create_fyi_txt_index(): }, ) save_to_red_bucket("fyi-index.txt", index) + + +## DirtyBits management for the RFC index + +RFCINDEX_SLUG = DirtyBits.Slugs.RFCINDEX + + +def mark_rfcindex_as_dirty(): + _, created = DirtyBits.objects.update_or_create( + slug=RFCINDEX_SLUG, defaults={"dirty_time": timezone.now()} + ) + if created: + log(f"Created DirtyBits(slug='{RFCINDEX_SLUG}')") + + +def mark_rfcindex_as_processed(when: datetime.datetime): + n_updated = DirtyBits.objects.filter( + Q(processed_time__isnull=True) | Q(processed_time__lt=when), + slug=RFCINDEX_SLUG, + ).update(processed_time=when) + if n_updated > 0: + log(f"processed_time is now {when.isoformat()}") + else: + log("processed_time not updated, no matching record found") + + +def rfcindex_is_dirty(): + """Does the rfc index need to be updated?""" + dirty_work, created = DirtyBits.objects.get_or_create( + slug=RFCINDEX_SLUG, defaults={"dirty_time": timezone.now()} + ) + if created: + log(f"Created DirtyBits(slug='{RFCINDEX_SLUG}')") + display_processed_time = ( + dirty_work.processed_time.isoformat() + if dirty_work.processed_time is not None + else "never" + ) + log( + f"DirtyBits(slug='{RFCINDEX_SLUG}'): " + f"dirty_time={dirty_work.dirty_time.isoformat()} " + f"processed_time={display_processed_time}" + ) + return ( + dirty_work.processed_time is None + or dirty_work.dirty_time >= dirty_work.processed_time + ) diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index c48368cccd..2805f431bf 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -24,6 +24,7 @@ create_rfc_txt_index, create_rfc_xml_index, create_std_txt_index, + rfcindex_is_dirty, mark_rfcindex_as_processed, ) from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper from ietf.utils import log @@ -288,33 +289,40 @@ def load_rfcs_into_blobdb_task(start: int, end: int): @shared_task -def create_rfc_index_task(): - try: - create_rfc_txt_index() - except Exception as e: - log.log(f"Error: failure in creating rfc-index.txt. {e}") - pass - - try: - create_rfc_xml_index() - except Exception as e: - log.log(f"Error: failure in creating rfc-index.xml. {e}") - pass - - try: - create_bcp_txt_index() - except Exception as e: - log.log(f"Error: failure in creating bcp-index.txt. {e}") - pass - - try: - create_std_txt_index() - except Exception as e: - log.log(f"Error: failure in creating std-index.txt. {e}") - pass - - try: - create_fyi_txt_index() - except Exception as e: - log.log(f"Error: failure in creating fyi-index.txt. {e}") - pass +def refresh_rfc_index_task(): + if rfcindex_is_dirty(): + # new_processed_time is the *start* of processing so that any changes after + # this point will trigger another refresh + new_processed_time = timezone.now() + + try: + create_rfc_txt_index() + except Exception as e: + log.log(f"Error: failure in creating rfc-index.txt. {e}") + pass + + try: + create_rfc_xml_index() + except Exception as e: + log.log(f"Error: failure in creating rfc-index.xml. {e}") + pass + + try: + create_bcp_txt_index() + except Exception as e: + log.log(f"Error: failure in creating bcp-index.txt. {e}") + pass + + try: + create_std_txt_index() + except Exception as e: + log.log(f"Error: failure in creating std-index.txt. {e}") + pass + + try: + create_fyi_txt_index() + except Exception as e: + log.log(f"Error: failure in creating fyi-index.txt. {e}") + pass + + mark_rfcindex_as_processed(new_processed_time) diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py index e6324ad7cd..cb8841cdc6 100644 --- a/ietf/utils/admin.py +++ b/ietf/utils/admin.py @@ -1,71 +1,30 @@ -# Copyright The IETF Trust 2011-2020, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2011-2026, All Rights Reserved from django.contrib import admin -from django.utils.encoding import force_str - -def name(obj): - if hasattr(obj, 'abbrev'): - return obj.abbrev() - elif hasattr(obj, 'name'): - if callable(obj.name): - name = obj.name() - else: - name = force_str(obj.name) - if name: - return name - return str(obj) - -def admin_link(field, label=None, ordering="", display=name, suffix=""): - if not label: - label = field.capitalize().replace("_", " ").strip() - if ordering == "": - ordering = field - def _link(self): - obj = self - for attr in field.split("__"): - obj = getattr(obj, attr) - if callable(obj): - obj = obj() - if hasattr(obj, "all"): - objects = obj.all() - elif callable(obj): - objects = obj() - if not hasattr(objects, "__iter__"): - objects = [ objects ] - elif hasattr(obj, "__iter__"): - objects = obj - else: - objects = [ obj ] - chunks = [] - for obj in objects: - app = obj._meta.app_label - model = obj.__class__.__name__.lower() - id = obj.pk - chunks += [ '%(display)s' % - {'app':app, "model": model, "id":id, "display": display(obj), "suffix":suffix, } ] - return ", ".join(chunks) - _link.allow_tags = True - _link.short_description = label - _link.admin_order_field = ordering - return _link +from .models import DumpInfo, DirtyBits class SaferStackedInline(admin.StackedInline): """StackedInline without delete by default""" + can_delete = False # no delete button show_change_link = True # show a link to the resource (where it can be deleted) class SaferTabularInline(admin.TabularInline): """TabularInline without delete by default""" + can_delete = False # no delete button show_change_link = True # show a link to the resource (where it can be deleted) -from .models import DumpInfo +@admin.register(DumpInfo) class DumpInfoAdmin(admin.ModelAdmin): - list_display = ['date', 'host', 'tz'] - list_filter = ['date'] -admin.site.register(DumpInfo, DumpInfoAdmin) + list_display = ["date", "host", "tz"] + list_filter = ["date"] + + +@admin.register(DirtyBits) +class DirtyBitsAdmin(admin.ModelAdmin): + list_display = ["slug", "dirty_time", "processed_time"] diff --git a/ietf/utils/migrations/0003_dirtybits.py b/ietf/utils/migrations/0003_dirtybits.py new file mode 100644 index 0000000000..11f6ed09f6 --- /dev/null +++ b/ietf/utils/migrations/0003_dirtybits.py @@ -0,0 +1,37 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("utils", "0002_delete_versioninfo"), + ] + + operations = [ + migrations.CreateModel( + name="DirtyBits", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "slug", + models.CharField( + choices=[("rfcindex", "RFC Index")], max_length=40, unique=True + ), + ), + ("dirty_time", models.DateTimeField(blank=True, null=True)), + ("processed_time", models.DateTimeField(blank=True, null=True)), + ], + options={ + "verbose_name_plural": "dirty bits", + }, + ), + ] diff --git a/ietf/utils/models.py b/ietf/utils/models.py index 21af5766e9..13afbdfe20 100644 --- a/ietf/utils/models.py +++ b/ietf/utils/models.py @@ -1,14 +1,35 @@ -# Copyright The IETF Trust 2015-2020, All Rights Reserved +# Copyright The IETF Trust 2015-2026, All Rights Reserved import itertools from django.db import models + +class DirtyBits(models.Model): + """A weak semaphore mechanism for coordination with celery beat tasks + + Web workers will set the "dirty_time" value for a given dirtybit slug. + Celery workers will do work if "processed_time" < "dirty_time" and update + "processed_time". + """ + + class Slugs(models.TextChoices): + RFCINDEX = "rfcindex", "RFC Index" + + # next line can become `...choices=Slugs)` when we get to Django 5.x + slug = models.CharField(max_length=40, blank=False, choices=Slugs.choices, unique=True) + dirty_time = models.DateTimeField(null=True, blank=True) + processed_time = models.DateTimeField(null=True, blank=True) + + class Meta: + verbose_name_plural = "dirty bits" + + class DumpInfo(models.Model): date = models.DateTimeField() host = models.CharField(max_length=128) tz = models.CharField(max_length=32, default='UTC') - + class ForeignKey(models.ForeignKey): "A local ForeignKey proxy which provides the on_delete value required under Django 2.0." def __init__(self, to, on_delete=models.CASCADE, **kwargs): diff --git a/ietf/utils/resources.py b/ietf/utils/resources.py index 1252cfef14..63206eb33a 100644 --- a/ietf/utils/resources.py +++ b/ietf/utils/resources.py @@ -1,6 +1,4 @@ -# Copyright The IETF Trust 2014-2019, All Rights Reserved -# -*- coding: utf-8 -*- -# Autogenerated by the mkresources management command 2014-11-13 05:39 +# Copyright The IETF Trust 2014-2026, All Rights Reserved from ietf.api import ModelResource @@ -12,7 +10,7 @@ from django.contrib.contenttypes.models import ContentType from ietf import api -from ietf.utils.models import DumpInfo +from ietf.utils.models import DirtyBits, DumpInfo class UserResource(ModelResource): @@ -43,3 +41,9 @@ class Meta: "host": ALL, } api.utils.register(DumpInfoResource()) + + +class DirtyBitsResource(ModelResource): + class Meta: + queryset = DirtyBits.objects.none() +api.utils.register(DirtyBitsResource())