feat: debounce rfc index refresh (ietf-tools#10708)

* feat: DirtyBits model + admin

* style: ruff + modernize utils/admin.py

* chore: remove unused code

* feat: rfcindex DirtyBits helpers

* feat: refresh RFC index only when dirty

Renames the task to better match its purpose

* test: update test

* fix: typo

* chore: add (empty) DirtyBits resource

* fix: actually call mark_rfcindex_as_processed

Author: jennifer-richards

ietf/api/tests_views_rpc.py

@@ -1,4 +1,5 @@
 # Copyright The IETF Trust 2025, All Rights Reserved
+import datetime
 from io import StringIO
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -10,12 +11,15 @@
 from django.test.utils import override_settings
 from django.urls import reverse as urlreverse
 import mock
+from django.utils import timezone
 
 from ietf.blobdb.models import Blob
 from ietf.doc.factories import IndividualDraftFactory, RfcFactory, WgDraftFactory, WgRfcFactory
 from ietf.doc.models import RelatedDocument, Document
 from ietf.group.factories import RoleFactory, GroupFactory
 from ietf.person.factories import PersonFactory
+from ietf.sync.rfcindex import rfcindex_is_dirty
+from ietf.utils.models import DirtyBits
 from ietf.utils.test_utils import APITestCase, reload_db_objects
 
 
@@ -408,16 +412,21 @@ def _valid_post_data():
             )
 
     @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
-    @mock.patch("ietf.api.views_rpc.create_rfc_index_task")
-    def test_refresh_rfc_index(self, mock_task):
+    def test_refresh_rfc_index(self):
+        DirtyBits.objects.create(
+            slug=DirtyBits.Slugs.RFCINDEX,
+            dirty_time=timezone.now() - datetime.timedelta(days=1),
+            processed_time=timezone.now() - datetime.timedelta(hours=12),
+        )
+        self.assertFalse(rfcindex_is_dirty())
         url = urlreverse("ietf.api.purple_api.refresh_rfc_index")
         response = self.client.get(url)
         self.assertEqual(response.status_code, 403)
         response = self.client.get(url, headers={"X-Api-Key": "invalid-token"})
         self.assertEqual(response.status_code, 403)
         response = self.client.get(url, headers={"X-Api-Key": "valid-token"})
         self.assertEqual(response.status_code, 405)
-        self.assertFalse(mock_task.delay.called)
+        self.assertFalse(rfcindex_is_dirty())
         response = self.client.post(url, headers={"X-Api-Key": "valid-token"})
         self.assertEqual(response.status_code, 202)
-        self.assertTrue(mock_task.delay.called)
+        self.assertTrue(rfcindex_is_dirty())

ietf/api/views_rpc.py

@@ -47,7 +47,7 @@
     update_rfc_searchindex_task,
 )
 from ietf.person.models import Email, Person
-from ietf.sync.tasks import create_rfc_index_task
+from ietf.sync.rfcindex import mark_rfcindex_as_dirty
 
 
 class Conflict(APIException):
@@ -548,5 +548,5 @@ class RfcIndexView(APIView):
         request=None,
     )
     def post(self, request):
-        create_rfc_index_task.delay()
+        mark_rfcindex_as_dirty()
         return Response(status=202)

ietf/sync/rfcindex.py

@@ -1,4 +1,5 @@
 # Copyright The IETF Trust 2026, All Rights Reserved
+import datetime
 import json
 from collections import defaultdict
 from collections.abc import Container
@@ -11,6 +12,7 @@
 
 from django.conf import settings
 from django.core.files.base import ContentFile
+from django.db.models import Q
 from lxml import etree
 
 from django.core.files.storage import storages
@@ -22,6 +24,7 @@
 from ietf.doc.models import Document
 from ietf.name.models import StdLevelName
 from ietf.utils.log import log
+from ietf.utils.models import DirtyBits
 
 FORMATS_FOR_INDEX = ["txt", "html", "pdf", "xml", "ps"]
 SS_TXT_MARGIN = 3
@@ -739,3 +742,50 @@ def create_fyi_txt_index():
         },
     )
     save_to_red_bucket("fyi-index.txt", index)
+
+
+## DirtyBits management for the RFC index
+
+RFCINDEX_SLUG = DirtyBits.Slugs.RFCINDEX
+
+
+def mark_rfcindex_as_dirty():
+    _, created = DirtyBits.objects.update_or_create(
+        slug=RFCINDEX_SLUG, defaults={"dirty_time": timezone.now()}
+    )
+    if created:
+        log(f"Created DirtyBits(slug='{RFCINDEX_SLUG}')")
+
+
+def mark_rfcindex_as_processed(when: datetime.datetime):
+    n_updated = DirtyBits.objects.filter(
+        Q(processed_time__isnull=True) | Q(processed_time__lt=when),
+        slug=RFCINDEX_SLUG,
+    ).update(processed_time=when)
+    if n_updated > 0:
+        log(f"processed_time is now {when.isoformat()}")
+    else:
+        log("processed_time not updated, no matching record found")
+
+
+def rfcindex_is_dirty():
+    """Does the rfc index need to be updated?"""
+    dirty_work, created = DirtyBits.objects.get_or_create(
+        slug=RFCINDEX_SLUG, defaults={"dirty_time": timezone.now()}
+    )
+    if created:
+        log(f"Created DirtyBits(slug='{RFCINDEX_SLUG}')")
+    display_processed_time = (
+        dirty_work.processed_time.isoformat()
+        if dirty_work.processed_time is not None
+        else "never"
+    )
+    log(
+        f"DirtyBits(slug='{RFCINDEX_SLUG}'): "
+        f"dirty_time={dirty_work.dirty_time.isoformat()} "
+        f"processed_time={display_processed_time}"
+    )
+    return (
+        dirty_work.processed_time is None
+        or dirty_work.dirty_time >= dirty_work.processed_time
+    )

ietf/sync/tasks.py

@@ -24,6 +24,7 @@
     create_rfc_txt_index,
     create_rfc_xml_index,
     create_std_txt_index,
+    rfcindex_is_dirty, mark_rfcindex_as_processed,
 )
 from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper
 from ietf.utils import log
@@ -288,33 +289,40 @@ def load_rfcs_into_blobdb_task(start: int, end: int):
 
 
 @shared_task
-def create_rfc_index_task():
-    try:
-        create_rfc_txt_index()
-    except Exception as e:
-        log.log(f"Error: failure in creating rfc-index.txt. {e}")
-        pass
-
-    try:
-        create_rfc_xml_index()
-    except Exception as e:
-        log.log(f"Error: failure in creating rfc-index.xml. {e}")
-        pass
-
-    try:
-        create_bcp_txt_index()
-    except Exception as e:
-        log.log(f"Error: failure in creating bcp-index.txt. {e}")
-        pass
-
-    try:
-        create_std_txt_index()
-    except Exception as e:
-        log.log(f"Error: failure in creating std-index.txt. {e}")
-        pass
-
-    try:
-        create_fyi_txt_index()
-    except Exception as e:
-        log.log(f"Error: failure in creating fyi-index.txt. {e}")
-        pass
+def refresh_rfc_index_task():
+    if rfcindex_is_dirty():
+        # new_processed_time is the *start* of processing so that any changes after
+        # this point will trigger another refresh
+        new_processed_time = timezone.now()
+
+        try:
+            create_rfc_txt_index()
+        except Exception as e:
+            log.log(f"Error: failure in creating rfc-index.txt. {e}")
+            pass
+
+        try:
+            create_rfc_xml_index()
+        except Exception as e:
+            log.log(f"Error: failure in creating rfc-index.xml. {e}")
+            pass
+
+        try:
+            create_bcp_txt_index()
+        except Exception as e:
+            log.log(f"Error: failure in creating bcp-index.txt. {e}")
+            pass
+
+        try:
+            create_std_txt_index()
+        except Exception as e:
+            log.log(f"Error: failure in creating std-index.txt. {e}")
+            pass
+
+        try:
+            create_fyi_txt_index()
+        except Exception as e:
+            log.log(f"Error: failure in creating fyi-index.txt. {e}")
+            pass
+
+        mark_rfcindex_as_processed(new_processed_time)

ietf/utils/admin.py

@@ -1,71 +1,30 @@
-# Copyright The IETF Trust 2011-2020, All Rights Reserved
-# -*- coding: utf-8 -*-
+# Copyright The IETF Trust 2011-2026, All Rights Reserved
 
 
 from django.contrib import admin
-from django.utils.encoding import force_str
-
-def name(obj):
-    if hasattr(obj, 'abbrev'):
-        return obj.abbrev()
-    elif hasattr(obj, 'name'):
-        if callable(obj.name):
-            name = obj.name()
-        else:
-            name = force_str(obj.name)
-        if name:
-            return name
-    return str(obj)
-    
-def admin_link(field, label=None, ordering="", display=name, suffix=""):
-    if not label:
-        label = field.capitalize().replace("_", " ").strip()
-    if ordering == "":
-        ordering = field
-    def _link(self):
-        obj = self
-        for attr in field.split("__"):
-            obj = getattr(obj, attr)
-            if callable(obj):
-                obj = obj()
-        if hasattr(obj, "all"):
-            objects = obj.all()
-        elif callable(obj):
-            objects = obj()
-            if not hasattr(objects, "__iter__"):
-                objects = [ objects ]
-        elif hasattr(obj, "__iter__"):
-            objects = obj
-        else:
-            objects = [ obj ]
-        chunks = []
-        for obj in objects:
-            app = obj._meta.app_label
-            model = obj.__class__.__name__.lower()
-            id = obj.pk
-            chunks += [ '<a href="/admin/%(app)s/%(model)s/%(id)s/%(suffix)s">%(display)s</a>' %
-                {'app':app, "model": model, "id":id, "display": display(obj), "suffix":suffix, } ]
-        return ", ".join(chunks)
-    _link.allow_tags = True
-    _link.short_description = label
-    _link.admin_order_field = ordering
-    return _link
+from .models import DumpInfo, DirtyBits
 
 
 class SaferStackedInline(admin.StackedInline):
     """StackedInline without delete by default"""
+
     can_delete = False  # no delete button
     show_change_link = True  # show a link to the resource (where it can be deleted)
 
 
 class SaferTabularInline(admin.TabularInline):
     """TabularInline without delete by default"""
+
     can_delete = False  # no delete button
     show_change_link = True  # show a link to the resource (where it can be deleted)
 
 
-from .models import DumpInfo
+@admin.register(DumpInfo)
 class DumpInfoAdmin(admin.ModelAdmin):
-    list_display = ['date', 'host', 'tz']
-    list_filter = ['date']
-admin.site.register(DumpInfo, DumpInfoAdmin)
+    list_display = ["date", "host", "tz"]
+    list_filter = ["date"]
+
+
+@admin.register(DirtyBits)
+class DirtyBitsAdmin(admin.ModelAdmin):
+    list_display = ["slug", "dirty_time", "processed_time"]

ietf/utils/migrations/0003_dirtybits.py

@@ -0,0 +1,37 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("utils", "0002_delete_versioninfo"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DirtyBits",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "slug",
+                    models.CharField(
+                        choices=[("rfcindex", "RFC Index")], max_length=40, unique=True
+                    ),
+                ),
+                ("dirty_time", models.DateTimeField(blank=True, null=True)),
+                ("processed_time", models.DateTimeField(blank=True, null=True)),
+            ],
+            options={
+                "verbose_name_plural": "dirty bits",
+            },
+        ),
+    ]

ietf/utils/models.py

@@ -1,14 +1,35 @@
-# Copyright The IETF Trust 2015-2020, All Rights Reserved
+# Copyright The IETF Trust 2015-2026, All Rights Reserved
 
 import itertools
 
 from django.db import models
 
+
+class DirtyBits(models.Model):
+    """A weak semaphore mechanism for coordination with celery beat tasks
+
+    Web workers will set the "dirty_time" value for a given dirtybit slug.
+    Celery workers will do work if "processed_time" < "dirty_time" and update
+    "processed_time".
+    """
+
+    class Slugs(models.TextChoices):
+        RFCINDEX = "rfcindex", "RFC Index"
+
+    # next line can become `...choices=Slugs)` when we get to Django 5.x
+    slug = models.CharField(max_length=40, blank=False, choices=Slugs.choices, unique=True)
+    dirty_time = models.DateTimeField(null=True, blank=True)
+    processed_time = models.DateTimeField(null=True, blank=True)
+
+    class Meta:
+        verbose_name_plural = "dirty bits"
+
+    
 class DumpInfo(models.Model):
     date = models.DateTimeField()
     host = models.CharField(max_length=128)
     tz   = models.CharField(max_length=32, default='UTC')
-    
+
 class ForeignKey(models.ForeignKey):
     "A local ForeignKey proxy which provides the on_delete value required under Django 2.0."
     def __init__(self, to, on_delete=models.CASCADE, **kwargs):