feat: Move IESG agenda items from filesystem to DB (ietf-tools#5366)

* feat: Add TelechatAgendaContent model and related support

* feat: Add UI for managing TelechatAgendaContents

* refactor: Rename _view view to _manage

* feat: Add a view to dump the TelechatAgendaContent as text/plain

* refactor: Point agenda_data() helpers at content in the DB

* refactor: Replace references to settings URLs/paths with new plumbing

* chore: Remove now-obsolete settings from settings.py

* feat: Link to telechat_agenda_content_manage view from iesg agenda

* fix: Use correct view name

* feat: Link from agenda content management page to IESG agenda view

* chore: Create resources

* chore: Add new names to names.json

* chore: Renumber migration after rebase

* chore: Remove unused import

* fix: Clean up partially removed code

* chore: Add admin model for TelechatAgendaContent

* chore: Simplify __str__ method for TelechatAgendaContent

* test: Add TelechatAgendaContentFactory

* test: Test the fill_in_agenda_administrivia() function

* test: Test that agenda contains action_items content

* test: Test that sensitive agenda links are restricted by role

* test: Test the telechat_agenda_content_view view

* test: Add test of telechat_agenda_content_edit view

* fix: Add type attribute to button to satisfy html validator

* test: Filter TelechatAgendaSectionName to used=True for tests

* test: More thoroughly test for likely(ish) permission errors

* fix: Fix typo in "tablist" role

* test: Test telechat_agenda_content_manage view

* style: Put back newlines at EOF

* chore: Add admin for TelechatAgendaSectionName

* chore: Renumber migrations

* fix: Depend on the correct migration

Forgot to update the number, but was also depending on the wrong
migration.

Author: jennifer-richards

ietf/iesg/admin.py

@@ -3,7 +3,7 @@
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import TelechatDocEvent
-from ietf.iesg.models import TelechatDate, TelechatAgendaItem
+from ietf.iesg.models import TelechatDate, TelechatAgendaItem, TelechatAgendaContent
 
 class TelechatAgendaItemAdmin(admin.ModelAdmin):
     pass
@@ -22,3 +22,6 @@ def save_model(self, request, obj, form, change):
 
 admin.site.register(TelechatDate, TelechatDateAdmin)
 
+class TelechatAgendaContentAdmin(admin.ModelAdmin):
+    list_display = ('section',)
+admin.site.register(TelechatAgendaContent, TelechatAgendaContentAdmin)

ietf/iesg/agenda.py

@@ -4,7 +4,6 @@
 
 # utilities for constructing agendas for IESG telechats
 
-import io
 import datetime
 from collections import OrderedDict
 
@@ -15,7 +14,7 @@
 
 from ietf.doc.models import Document, LastCallDocEvent, ConsensusDocEvent
 from ietf.doc.utils_search import fill_in_telechat_date
-from ietf.iesg.models import TelechatDate, TelechatAgendaItem
+from ietf.iesg.models import TelechatDate, TelechatAgendaItem, TelechatAgendaContent
 from ietf.review.utils import review_assignments_to_list_for_docs
 from ietf.utils.timezone import date_today, make_aware
 
@@ -140,20 +139,18 @@ def agenda_sections():
         ])
 
 def fill_in_agenda_administrivia(date, sections):
-    extra_info_files = (
-        ("1.1", "roll_call", settings.IESG_ROLL_CALL_FILE),
-        ("1.3", "minutes", settings.IESG_MINUTES_FILE),
-        ("1.4", "action_items", settings.IESG_TASK_FILE),
-        )
+    extra_info = (
+        ("1.1", "roll_call"),
+        ("1.3", "minutes"),
+        ("1.4", "action_items"),
+    )
 
-    for s, key, filename in extra_info_files:
+    for s, key in extra_info:
         try:
-            with io.open(filename, 'r', encoding='utf-8', errors='replace') as f:
-                t = f.read().strip()
-        except IOError:
-            t = "(Error reading %s)" % filename
-
-        sections[s]["text"] = t
+            text = TelechatAgendaContent.objects.get(section__slug=key).text
+        except TelechatAgendaContent.DoesNotExist:
+            text = ""
+        sections[s]["text"] = text
 
 def fill_in_agenda_docs(date, sections, docs=None):
     if not docs:

ietf/iesg/factories.py

@@ -4,7 +4,7 @@
 import debug    # pyflakes:ignore
 import factory
 
-from ietf.iesg.models import TelechatAgendaItem
+from ietf.iesg.models import TelechatAgendaItem, TelechatAgendaContent
 
 
 class IESGMgmtItemFactory(factory.django.DjangoModelFactory):
@@ -14,3 +14,10 @@ class Meta:
     type = 3
     text = factory.Faker('paragraph', nb_sentences=3)
     title = factory.Faker('sentence', nb_words=3)
+
+
+class TelechatAgendaContentFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = TelechatAgendaContent
+
+    text = factory.Faker('paragraph', nb_sentences=5)

ietf/iesg/migrations/0002_telechatagendacontent.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.28 on 2023-03-10 16:47
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name', '0002_telechatagendasectionname'),
+        ('iesg', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TelechatAgendaContent',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('text', models.TextField(blank=True)),
+                ('section', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='name.TelechatAgendaSectionName')),
+            ],
+        ),
+    ]

ietf/iesg/models.py

@@ -39,6 +39,7 @@
 from django.conf import settings
 from django.db import models
 
+from ietf.name.models import TelechatAgendaSectionName
 from ietf.utils.timezone import date_today
 
 
@@ -95,3 +96,11 @@ class Meta:
         indexes = [
             models.Index(fields=['-date',]),
         ]
+
+
+class TelechatAgendaContent(models.Model):
+    section = models.ForeignKey(TelechatAgendaSectionName, on_delete=models.PROTECT)
+    text = models.TextField(blank=True)
+
+    def __str__(self):
+        return f"{self.section.name} content"

ietf/iesg/resources.py

@@ -3,13 +3,13 @@
 # Autogenerated by the mkresources management command 2014-11-13 23:53
 
 
-from ietf.api import ModelResource
-from tastypie.constants import ALL
+from ietf.api import ModelResource, ToOneField
+from tastypie.constants import ALL, ALL_WITH_RELATIONS
 from tastypie.cache import SimpleCache
 
 from ietf import api
 
-from ietf.iesg.models import TelechatDate, Telechat, TelechatAgendaItem
+from ietf.iesg.models import TelechatDate, Telechat, TelechatAgendaItem, TelechatAgendaContent
 
 
 class TelechatDateResource(ModelResource):
@@ -59,3 +59,20 @@ class Meta:
         }
 api.iesg.register(TelechatAgendaItemResource())
 
+
+
+from ietf.name.resources import TelechatAgendaSectionNameResource
+class TelechatAgendaContentResource(ModelResource):
+    section          = ToOneField(TelechatAgendaSectionNameResource, 'section')
+    class Meta:
+        queryset = TelechatAgendaContent.objects.none()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'telechatagendacontent'
+        ordering = ['id', ]
+        filtering = { 
+            "id": ALL,
+            "text": ALL,
+            "section": ALL_WITH_RELATIONS,
+        }
+api.iesg.register(TelechatAgendaContentResource())

ietf/iesg/tests.py

@@ -22,12 +22,12 @@
 from ietf.doc.utils import create_ballot_if_not_open
 from ietf.group.factories import RoleFactory, GroupFactory
 from ietf.group.models import Group, GroupMilestone, Role
-from ietf.iesg.agenda import get_agenda_date, agenda_data
-from ietf.iesg.models import TelechatDate
-from ietf.name.models import StreamName
+from ietf.iesg.agenda import get_agenda_date, agenda_data, fill_in_agenda_administrivia, agenda_sections
+from ietf.iesg.models import TelechatDate, TelechatAgendaContent
+from ietf.name.models import StreamName, TelechatAgendaSectionName
 from ietf.person.models import Person
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent
-from ietf.iesg.factories import IESGMgmtItemFactory
+from ietf.iesg.factories import IESGMgmtItemFactory, TelechatAgendaContentFactory
 from ietf.utils.timezone import date_today, DEADLINE_TZINFO
 
 
@@ -141,6 +141,16 @@ def setUp(self):
         for i in range(0, 10):
             self.mgmt_items.append(IESGMgmtItemFactory())
 
+    def test_fill_in_agenda_administrivia(self):
+        roll_call = TelechatAgendaContentFactory(section_id='roll_call')
+        minutes = TelechatAgendaContentFactory(section_id='minutes')
+        action_items = TelechatAgendaContentFactory(section_id='action_items')
+        sections = agenda_sections()
+        fill_in_agenda_administrivia(None, sections)  # n.b., date parameter is unused at present
+        self.assertIn(roll_call.text, sections["1.1"]["text"])
+        self.assertIn(minutes.text, sections["1.3"]["text"])
+        self.assertIn(action_items.text, sections["1.4"]["text"])
+
     def test_fill_in_agenda_docs(self):
         draft = self.telechat_docs["ietf_draft"]
         statchg = self.telechat_docs["statchg"]
@@ -337,9 +347,12 @@ def test_agenda_json(self):
         self.assertTrue(r.json())
 
     def test_agenda(self):
+        action_items = TelechatAgendaContentFactory(section_id='action_items')
         r = self.client.get(urlreverse("ietf.iesg.views.agenda"))
         self.assertEqual(r.status_code, 200)
 
+        self.assertContains(r, action_items.text)
+
         for k, d in self.telechat_docs.items():
             if d.type_id == "charter":
                 self.assertContains(r, d.group.name, msg_prefix="%s '%s' not in response" % (k, d.group.name))
@@ -356,6 +369,29 @@ def test_agenda(self):
         # Make sure the sort places 6.9 before 6.10
         self.assertLess(r.content.find(b"6.9"), r.content.find(b"6.10"))
 
+    def test_agenda_restricted_sections(self):
+        r = self.client.get(urlreverse("ietf.iesg.views.agenda"))
+        # not logged in
+        for section_id in ("roll_call", "minutes"):
+            self.assertNotContains(
+                r, urlreverse("ietf.iesg.views.telechat_agenda_content_view", kwargs={"section": section_id})
+            )
+
+        self.client.login(username="plain", password="plain+password")
+        for section_id in ("roll_call", "minutes"):
+            self.assertNotContains(
+                r, urlreverse("ietf.iesg.views.telechat_agenda_content_view", kwargs={"section": section_id})
+            )
+
+        for username in ("ad", "secretary", "iab chair"):
+            self.client.login(username=username, password=f"{username}+password")
+            r = self.client.get(urlreverse("ietf.iesg.views.agenda"))
+            self.assertEqual(r.status_code, 200)
+            for section_id in ("roll_call", "minutes"):
+                self.assertContains(
+                    r, urlreverse("ietf.iesg.views.telechat_agenda_content_view", kwargs={"section": section_id})
+                )
+
     def test_agenda_txt(self):
         r = self.client.get(urlreverse("ietf.iesg.views.agenda_txt"))
         self.assertEqual(r.status_code, 200)
@@ -549,3 +585,104 @@ def test_reschedule(self):
         self.assertEqual(draft.latest_event(TelechatDocEvent, "scheduled_for_telechat").telechat_date, d)
         self.assertTrue(not draft.latest_event(TelechatDocEvent, "scheduled_for_telechat").returning_item)
         self.assertEqual(draft.docevent_set.count(), events_before + 1)
+
+
+class TelechatAgendaContentTests(TestCase):
+    def test_telechat_agenda_content_view(self):
+        self.client.login(username="ad", password="ad+password")
+        r = self.client.get(urlreverse("ietf.iesg.views.telechat_agenda_content_view", kwargs={"section": "fake"}))
+        self.assertEqual(r.status_code, 404, "Nonexistent section should 404")
+        for section in TelechatAgendaSectionName.objects.filter(used=True).values_list("slug", flat=True):
+            r = self.client.get(
+                urlreverse("ietf.iesg.views.telechat_agenda_content_view", kwargs={"section": section})
+            )
+            self.assertEqual(r.status_code, 404, "Section with no content should 404")
+        for section in TelechatAgendaSectionName.objects.filter(used=True).values_list("slug", flat=True):
+            content = TelechatAgendaContentFactory(section_id=section).text
+            r = self.client.get(
+                urlreverse("ietf.iesg.views.telechat_agenda_content_view", kwargs={"section": section})
+            )
+            self.assertContains(r, content, status_code=200)
+            self.assertEqual(r.get("Content-Type", None), "text/plain")
+
+    def test_telechat_agenda_content_view_permissions(self):
+        for section in TelechatAgendaSectionName.objects.filter(used=True).values_list("slug", flat=True):
+            TelechatAgendaContentFactory(section_id=section)
+            url = urlreverse("ietf.iesg.views.telechat_agenda_content_view", kwargs={"section": section})
+            self.client.logout()
+            login_testing_unauthorized(self, "plain", url)
+            login_testing_unauthorized(self, "ad", url)
+            self.assertEqual(self.client.get(url).status_code, 200)
+            self.client.login(username="iab chair", password="iab chair+password")
+            self.assertEqual(self.client.get(url).status_code, 200)
+            self.client.login(username="secretary", password="secretary+password")
+            self.assertEqual(self.client.get(url).status_code, 200)
+
+    def test_telechat_agenda_content_edit(self):
+        for section in TelechatAgendaSectionName.objects.filter(used=True):
+            self.assertFalse(TelechatAgendaContent.objects.filter(section=section).exists())
+            url = urlreverse("ietf.iesg.views.telechat_agenda_content_edit", kwargs={"section": section.slug})
+            self.client.logout()
+            login_testing_unauthorized(self, "plain", url, method="get")
+            login_testing_unauthorized(self, "ad", url, method="get")
+            login_testing_unauthorized(self, "iab chair", url, method="get")
+            login_testing_unauthorized(self, "secretary", url, method="get")
+            r = self.client.get(url)
+            self.assertContains(r, str(section), status_code=200)
+
+            self.client.logout()
+            login_testing_unauthorized(self, "plain", url, method="post")
+            login_testing_unauthorized(self, "ad", url, method="post")
+            login_testing_unauthorized(self, "iab chair", url, method="post")
+            login_testing_unauthorized(self, "secretary", url, method="post")
+            r = self.client.post(url, {"text": "This is some content"})
+            self.assertRedirects(r, urlreverse("ietf.iesg.views.telechat_agenda_content_manage"))
+            contents = TelechatAgendaContent.objects.filter(section=section)
+            self.assertEqual(contents.count(), 1)
+            self.assertEqual(contents.first().text, "This is some content")
+
+            self.client.logout()
+            login_testing_unauthorized(self, "plain", url, method="post")
+            login_testing_unauthorized(self, "ad", url, method="post")
+            login_testing_unauthorized(self, "iab chair", url, method="post")
+            login_testing_unauthorized(self, "secretary", url, method="post")
+            r = self.client.post(url, {"text": "This is some different content"})
+            self.assertRedirects(r, urlreverse("ietf.iesg.views.telechat_agenda_content_manage"))
+            contents = TelechatAgendaContent.objects.filter(section=section)
+            self.assertEqual(contents.count(), 1)
+            self.assertEqual(contents.first().text, "This is some different content")
+
+    def test_telechat_agenda_content_manage(self):
+        url = urlreverse("ietf.iesg.views.telechat_agenda_content_manage")
+        login_testing_unauthorized(self, "plain", url)
+        login_testing_unauthorized(self, "ad", url)
+        login_testing_unauthorized(self, "iab chair", url)
+        login_testing_unauthorized(self, "secretary", url)
+        self.assertEqual(TelechatAgendaContent.objects.count(), 0)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        pq = PyQuery(r.content)
+        for section in TelechatAgendaSectionName.objects.filter(used=True):
+            # check that there's a tab even when section is empty
+            nav_button = pq(f"button.nav-link#{section.slug}-tab")
+            self.assertEqual(nav_button.text(), str(section))
+            edit_url = urlreverse("ietf.iesg.views.telechat_agenda_content_edit", kwargs={"section": section.pk})
+            edit_button = pq(f'div#{section.slug} a[href="{edit_url}"]')
+            self.assertEqual(len(edit_button), 1)
+            self.assertIn(f"No {section}", pq(f"div#{section.slug}").text())
+            # and create a section for the next test
+            TelechatAgendaContentFactory(section_id=section.slug)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        pq = PyQuery(r.content)
+        for section in TelechatAgendaSectionName.objects.filter(used=True):
+            # check that there's a tab with the content
+            nav_button = pq(f"button.nav-link#{section.slug}-tab")
+            self.assertEqual(nav_button.text(), str(section))
+            edit_url = urlreverse("ietf.iesg.views.telechat_agenda_content_edit", kwargs={"section": section.pk})
+            edit_button = pq(f'div#{section.slug} a[href="{edit_url}"]')
+            self.assertEqual(len(edit_button), 1)
+            self.assertIn(
+                TelechatAgendaContent.objects.get(section=section).text, pq(f"div#{section.slug}").text()
+            )
+

ietf/iesg/urls.py

@@ -52,6 +52,9 @@
 
     url(r'^agenda/documents.txt$', views.agenda_documents_txt),
     url(r'^agenda/documents/$', views.agenda_documents),
+    url(r'^agenda/sections$', views.telechat_agenda_content_manage),
+    url(r'^agenda/section/(?P<section>[a-z_]+)$', views.telechat_agenda_content_view),
+    url(r'^agenda/section/(?P<section>[a-z_]+)/edit$', views.telechat_agenda_content_edit),
     url(r'^past/documents/$', views.past_documents),
     url(r'^agenda/telechat-(?:%(date)s-)?docs.tgz' % settings.URL_REGEXPS, views.telechat_docs_tarfile),
     url(r'^discusses/$', views.discusses),