Port IESG agenda pages to new schema, add tests for them, move agenda

utilities to agenda.py, fix a couple of things, remove obsolete test
versions of agenda view, add support for viewing future IESG agendas
through /agenda/YYYY-MM-DD/* (useful when testing)
 - Legacy-Id: 6395

Author: OleLaursen

ietf/iesg/agenda.py

@@ -0,0 +1,170 @@
+# utilities for constructing agendas for IESG telechats
+
+import codecs, re, os, datetime
+
+from django.http import Http404
+from django.conf import settings
+
+from ietf.iesg.models import TelechatDate, TelechatAgendaItem
+from ietf.doc.models import Document, TelechatDocEvent, LastCallDocEvent, ConsensusDocEvent, DocEvent
+from ietf.group.models import Group, GroupMilestone
+
+def get_agenda_date(date=None):
+    if not date:
+        try:
+            return TelechatDate.objects.active().order_by('date')[0].date
+        except IndexError:
+            return datetime.date.today()
+    else:
+        try:
+            return TelechatDate.objects.active().get(date=datetime.datetime.strptime(date, "%Y-%m-%d").date()).date
+        except (ValueError, TelechatDate.DoesNotExist):
+            raise Http404
+
+def get_doc_section(doc):
+    if doc.type_id == 'draft':
+        if doc.intended_std_level_id in ["bcp", "ds", "ps", "std"]:
+            s = "2"
+        else:
+            s = "3"
+
+        g = doc.group_acronym()
+        if g and str(g) != 'none':
+            s = s + "1"
+        elif (s == "3") and doc.stream_id in ("ise","irtf"):
+            s = s + "3"
+        else:
+            s = s + "2"
+        if doc.get_state_slug() != "rfc" and doc.get_state_slug('draft-iesg') not in ("lc", "writeupw", "goaheadw", "iesg-eva", "defer"):
+            s = s + "3"
+        elif doc.returning_item():
+            s = s + "2"
+        else:
+            s = s + "1"
+    elif doc.type_id == 'charter':
+        s = get_wg_section(doc.group)
+    elif doc.type_id == 'statchg':
+        protocol_action = False
+        for relation in doc.relateddocument_set.filter(relationship__slug__in=('tops','tois','tohist','toinf','tobcp','toexp')):
+            if relation.relationship.slug in ('tops','tois') or relation.target.document.std_level.slug in ('std','ds','ps'):
+                protocol_action = True
+        if protocol_action:
+            s="23"
+        else:
+            s="33"
+        if doc.get_state_slug() not in ("iesgeval", "defer", "appr-pr", "appr-pend", "appr-sent"):
+            s = s + "3"
+        elif doc.returning_item():
+            s = s + "2"
+        else:
+            s = s + "1"
+    elif doc.type_id == 'conflrev':
+        if doc.get_state('conflrev').slug not in ('adrev','iesgeval','appr-reqnopub-pend','appr-reqnopub-sent','appr-noprob-pend','appr-noprob-sent','defer'):
+             s = "343"
+        elif doc.returning_item():
+             s = "342"
+        else:
+             s = "341"
+
+    return s
+
+def get_wg_section(wg):
+    s = ""
+    charter_slug = None
+    if wg.charter:
+        charter_slug = wg.charter.get_state_slug()
+    if wg.state_id in ['active','dormant']:
+        if charter_slug in ['extrev','iesgrev']:
+            s = '422'
+        else:
+            s = '421'
+    else:
+        if charter_slug in ['extrev','iesgrev']:
+            s = '412'
+        else:
+            s = '411'
+    return s
+
+def agenda_docs(date):
+    matches = Document.objects.filter(docevent__telechatdocevent__telechat_date=date).select_related("stream").distinct()
+
+    docmatches = []
+        
+    for doc in matches:
+        if doc.latest_event(TelechatDocEvent, type="scheduled_for_telechat").telechat_date != date:
+            continue
+
+        e = doc.latest_event(type="started_iesg_process")
+        doc.balloting_started = e.time if e else datetime.datetime.min
+
+        if doc.type_id == "draft":
+            s = doc.get_state("draft-iana-review")
+            if s: # and s.slug in ("not-ok", "changed", "need-rev"):
+                doc.iana_review_state = str(s)
+
+            if doc.get_state_slug("draft-iesg") == "lc":
+                e = doc.latest_event(LastCallDocEvent, type="sent_last_call")
+                if e:
+                    doc.lastcall_expires = e.expires
+
+            if doc.stream_id in ("ietf", "irtf", "iab"):
+                doc.consensus = "Unknown"
+                e = doc.latest_event(ConsensusDocEvent, type="changed_consensus")
+                if e:
+                    doc.consensus = "Yes" if e.consensus else "No"
+        elif doc.type_id=='conflrev':
+            doc.conflictdoc = doc.relateddocument_set.get(relationship__slug='conflrev').target.document
+
+        docmatches.append(doc)
+
+    # Be careful to keep this the same as what's used in agenda_documents
+    docmatches.sort(key=lambda d: d.balloting_started)
+    
+    res = dict(("s%s%s%s" % (i, j, k), []) for i in range(2, 5) for j in range (1, 4) for k in range(1, 4))
+    for k in range(1,4):
+        res['s34%d'%k]=[]
+    for doc in docmatches:
+        section_key = "s" + get_doc_section(doc)
+        if section_key not in res:
+            res[section_key] = []
+        res[section_key].append(doc)
+    return res
+
+def agenda_wg_actions(date):
+    res = dict(("s%s%s%s" % (i, j, k), []) for i in range(2, 5) for j in range (1, 4) for k in range(1, 4))
+    charters = Document.objects.filter(type="charter", docevent__telechatdocevent__telechat_date=date).select_related("group").distinct()
+    charters = charters.filter(group__state__slug__in=["proposed","active"])
+    for c in charters:
+        if c.latest_event(TelechatDocEvent, type="scheduled_for_telechat").telechat_date != date:
+            continue
+
+        c.group.txt_link = settings.CHARTER_TXT_URL + "%s-%s.txt" % (c.canonical_name(), c.rev)
+
+        section_key = "s" + get_wg_section(c.group)
+        if section_key not in res:
+            res[section_key] = []
+        res[section_key].append(c)
+    return res
+
+def agenda_management_issues(date):
+    return TelechatAgendaItem.objects.filter(type=3).order_by('id')
+
+def agenda_data(request, date=None):
+    """Return a dict with the different IESG telechat agenda components."""
+    date = get_agenda_date(date)
+    docs = agenda_docs(date)
+    mgmt = agenda_management_issues(date)
+    wgs = agenda_wg_actions(date)
+    data = {'date':str(date), 'docs':docs,'mgmt':mgmt,'wgs':wgs}
+    for key, filename in {'action_items':settings.IESG_TASK_FILE,
+                          'roll_call':settings.IESG_ROLL_CALL_FILE,
+                          'minutes':settings.IESG_MINUTES_FILE}.items():
+        try:
+            f = codecs.open(filename, 'r', 'utf-8', 'replace')
+            text = f.read().strip()
+            f.close()
+            data[key] = text
+        except IOError:
+            data[key] = "(Error reading "+key+")"
+    return data
+

ietf/iesg/tests.py

@@ -1,17 +1,19 @@
-import os, shutil
+import os, shutil, json
 
 import django.test
 from django.core.urlresolvers import reverse as urlreverse
 from django.conf import settings
 
 from pyquery import PyQuery
 
+from ietf.utils.test_utils import SimpleUrlTestCase, RealDatabaseTest, canonicalize_feed, login_testing_unauthorized
 from ietf.utils.test_data import make_test_data
 from ietf.doc.models import Document, DocEvent, TelechatDocEvent, State
 from ietf.person.models import Person
 from ietf.group.models import Group
+from ietf.name.models import StreamName
 from ietf.iesg.models import *
-from ietf.utils.test_utils import SimpleUrlTestCase, RealDatabaseTest, canonicalize_feed, login_testing_unauthorized
+from ietf.iesg.agenda import get_agenda_date
 
 class ReviewDecisionsTests(django.test.TestCase):
     def test_review_decisions(self):
@@ -30,27 +32,113 @@ def test_review_decisions(self):
 
 
 class IESGAgendaTests(django.test.TestCase):
-    def test_feed(self):
-        draft = make_test_data()
+    def setUp(self):
+        make_test_data()
+
+        ise_draft = Document.objects.get(name="draft-imaginary-independent-submission")
+        ise_draft.stream = StreamName.objects.get(slug="ise")
+        ise_draft.save()
+
+        self.telechat_docs = {
+            "ietf_draft": Document.objects.get(name="draft-ietf-mars-test"),
+            "ise_draft": ise_draft,
+            "conflrev": Document.objects.get(name="conflict-review-imaginary-irtf-submission"),
+            "statusch": Document.objects.get(name="status-change-imaginary-mid-review"),
+            "charter": Document.objects.filter(type="charter")[0],
+            }
+
+        by = Person.objects.get(name="Aread Irector")
+        date = get_agenda_date()
+
+        for d in self.telechat_docs.values():
+            TelechatDocEvent.objects.create(type="scheduled_for_telechat",
+                                            doc=d,
+                                            by=by,
+                                            telechat_date=date,
+                                            returning_item=True)
 
+    def test_feed(self):
         url = "/feed/iesg-agenda/"
 
         r = self.client.get(url)
         self.assertEquals(r.status_code, 200)
-        self.assertTrue(draft.name not in r.content)
 
-        # add to schedule
-        e = TelechatDocEvent(type="scheduled_for_telechat")
-        e.doc = draft
-        e.by = Person.objects.get(name="Aread Irector")
-        e.telechat_date = TelechatDate.objects.active()[0].date
-        e.returning_item = True
-        e.save()
+        for d in self.telechat_docs.values():
+            self.assertTrue(d.name in r.content)
+            self.assertTrue(d.title in r.content)
+
+    def test_agenda_json(self):
+        r = self.client.get(urlreverse("ietf.iesg.views.agenda_json"))
+        self.assertEquals(r.status_code, 200)
 
+        for k, d in self.telechat_docs.iteritems():
+            if d.type_id == "charter":
+                self.assertTrue(d.group.name in r.content, "%s not in response" % k)
+                self.assertTrue(d.group.acronym in r.content, "%s acronym not in response" % k)
+            else:
+                self.assertTrue(d.name in r.content, "%s not in response" % k)
+                self.assertTrue(d.title in r.content, "%s title not in response" % k)
+
+        self.assertTrue(json.loads(r.content))
+
+    def test_agenda(self):
+        r = self.client.get(urlreverse("ietf.iesg.views.agenda"))
+        self.assertEquals(r.status_code, 200)
+
+        for k, d in self.telechat_docs.iteritems():
+            self.assertTrue(d.name in r.content, "%s not in response" % k)
+            self.assertTrue(d.title in r.content, "%s title not in response" % k)
+
+    def test_agenda_txt(self):
+        r = self.client.get(urlreverse("ietf.iesg.views.agenda_txt"))
+        self.assertEquals(r.status_code, 200)
+
+        for k, d in self.telechat_docs.iteritems():
+            if d.type_id == "charter":
+                self.assertTrue(d.group.name in r.content, "%s not in response" % k)
+                self.assertTrue(d.group.acronym in r.content, "%s acronym not in response" % k)
+            else:
+                self.assertTrue(d.name in r.content, "%s not in response" % k)
+                self.assertTrue(d.title in r.content, "%s title not in response" % k)
+
+    def test_agenda_scribe_template(self):
+        r = self.client.get(urlreverse("ietf.iesg.views.agenda_scribe_template"))
+        self.assertEquals(r.status_code, 200)
+
+        for k, d in self.telechat_docs.iteritems():
+            if d.type_id == "charter":
+                continue # scribe template doesn't contain chartering info
+
+            self.assertTrue(d.name in r.content, "%s not in response" % k)
+            self.assertTrue(d.title in r.content, "%s title not in response" % k)
+
+    def test_agenda_moderator_package(self):
+        url = urlreverse("ietf.iesg.views.agenda_moderator_package")
+        login_testing_unauthorized(self, "secretary", url)
         r = self.client.get(url)
         self.assertEquals(r.status_code, 200)
-        self.assertTrue(draft.name in r.content)
-        self.assertTrue(draft.title in r.content)
+
+        for k, d in self.telechat_docs.iteritems():
+            if d.type_id == "charter":
+                self.assertTrue(d.group.name in r.content, "%s not in response" % k)
+                self.assertTrue(d.group.acronym in r.content, "%s acronym not in response" % k)
+            else:
+                self.assertTrue(d.name in r.content, "%s not in response" % k)
+                self.assertTrue(d.title in r.content, "%s title not in response" % k)
+
+    def test_agenda_package(self):
+        url = urlreverse("ietf.iesg.views.agenda_package")
+        login_testing_unauthorized(self, "secretary", url)
+        r = self.client.get(url)
+        self.assertEquals(r.status_code, 200)
+
+        for k, d in self.telechat_docs.iteritems():
+            if d.type_id == "charter":
+                self.assertTrue(d.group.name in r.content, "%s not in response" % k)
+                self.assertTrue(d.group.acronym in r.content, "%s acronym not in response" % k)
+            else:
+                self.assertTrue(d.name in r.content, "%s not in response" % k)
+                self.assertTrue(d.title in r.content, "%s title not in response" % k)
 
 class RescheduleOnAgendaTestCase(django.test.TestCase):
     def test_reschedule(self):

ietf/iesg/urls.py

@@ -38,28 +38,22 @@
 
 urlpatterns = patterns('',
     (r'^telechat/.*$', 'django.views.generic.simple.redirect_to', { 'url': 'http://www.ietf.org/iesg/minutes.html' }),
-    (r'^decisions/(?:(?P<year>[0-9]{4})/)?$', views.review_decisions),
     (r'^ann/(?:ind|new|prev)/$', 'django.views.generic.simple.redirect_to', { 'url': "/iesg/decisions/", 'permanent': True }),
-    (r'^agenda/$', views.agenda),
-    (r'^agenda/agenda.txt$', views.agenda_txt),                        
-    (r'^agenda/agenda.json$', views.agenda_json),                        
-    (r'^agenda/scribe_template.html$', views.agenda_scribe_template),
-    (r'^agenda/moderator_package.html$', views.agenda_moderator_package),
-    (r'^agenda/agenda_package.txt$', views.agenda_package),
+
+    (r'^decisions/(?:(?P<year>[0-9]{4})/)?$', views.review_decisions),
+    (r'^agenda/(?:(?P<date>\d{4}-\d{2}-\d{2})/)?$', views.agenda),
+    (r'^agenda/(?:(?P<date>\d{4}-\d{2}-\d{2})/)?agenda.txt$', views.agenda_txt),
+    (r'^agenda/(?:(?P<date>\d{4}-\d{2}-\d{2})/)?agenda.json$', views.agenda_json),
+    (r'^agenda/(?:(?P<date>\d{4}-\d{2}-\d{2})/)?scribe_template.html$', views.agenda_scribe_template),
+    (r'^agenda/(?:(?P<date>\d{4}-\d{2}-\d{2})/)?moderator_package.html$', views.agenda_moderator_package),
+    (r'^agenda/(?:(?P<date>\d{4}-\d{2}-\d{2})/)?agenda_package.txt$', views.agenda_package),
     (r'^agenda/documents.txt$', views.agenda_documents_txt),
     (r'^agenda/documents/$', views.agenda_documents),
     (r'^agenda/telechat-(?P<year>\d+)-(?P<month>\d+)-(?P<day>\d+)-docs.tgz', views.telechat_docs_tarfile),
     (r'^discusses/$', views.discusses),
-    (r'^milestones', views.milestones_needing_review),
+    (r'^milestones$', views.milestones_needing_review),
     (r'^telechatdates/$', 'django.views.generic.simple.redirect_to', { 'url': '/admin/iesg/telechatdate/' }),
     url(r'^wgactions/$', views.working_group_actions, name="iesg_working_group_actions"),
     url(r'^wgactions/add/$', views.edit_working_group_action, { 'wga_id': None }, name="iesg_add_working_group_action"),
     url(r'^wgactions/(?P<wga_id>\d+)/$', views.edit_working_group_action, name="iesg_edit_working_group_action"),
 )
-
-if settings.SERVER_MODE != 'production':
-    urlpatterns += patterns('',
-        (r'^agenda/(?P<date>\d{4}-\d\d-\d\d)/$', views.agenda),
-        (r'^_test/moderator_package.html$', views.agenda_moderator_package_test),
-        (r'^_test/agenda_package.txt', views.agenda_package_test),
-    )