Summary: Make the meeting agenda code use the historic info available

for groups. In the process, added some select_related/prefetch_related
code to cut the number of database queries in the agenda down from
4000-5000 to about 50. Rewritten agenda CSV code to use Python arrays
rather than a template for improved readability/reuse. Rewritten week
view to assemble its data in Python. Plus a few other minor cleanups,
e.g. lame_description is now gone.

Fixes ietf-tools#1088. Branch ready for merge.
 - Legacy-Id: 9669

Author: OleLaursen

ietf/meeting/helpers.py

@@ -2,21 +2,26 @@
 
 import datetime
 import pytz
+import os
+import re
+from tempfile import mkstemp
 
 from django.http import HttpRequest
-from django.db.models import Max, Q
+from django.db.models import Max, Q, Prefetch, F
 from django.conf import settings
 from django.core.cache import cache
 from django.utils.cache import get_cache_key
 from django.shortcuts import get_object_or_404
 
 import debug                            # pyflakes:ignore
 
+from ietf.doc.models import Document
 from ietf.group.models import Group
 from ietf.ietfauth.utils import has_role, user_is_person
 from ietf.person.models  import Person
 from ietf.meeting.models import Meeting
-from ietf.utils.history import find_history_active_at
+from ietf.utils.history import find_history_active_at, find_history_replacements_active_at
+from ietf.utils.pipe import pipe
 
 def find_ads_for_meeting(meeting):
     ads = []
@@ -144,6 +149,109 @@ def meeting_updated(meeting):
     ts = tz.localize(ts)
     return ts
 
+def preprocess_assignments_for_agenda(assignments_queryset, meeting):
+    # prefetch some stuff to prevent a myriad of small, inefficient queries
+    assignments_queryset = assignments_queryset.select_related(
+        "timeslot", "timeslot__location", "timeslot__type",
+        "session",
+        "session__group", "session__group__charter", "session__group__charter__group",
+    ).prefetch_related(
+        Prefetch("session__materials",
+                 queryset=Document.objects.exclude(states__type=F("type"),states__slug='deleted').select_related("group").order_by("order"),
+                 to_attr="prefetched_active_materials",
+             ),
+        "timeslot__meeting",
+    )
+
+    assignments = list(assignments_queryset) # make sure we're set in stone
+
+    meeting_time = datetime.datetime.combine(meeting.date, datetime.time())
+
+    # replace groups with historic counterparts
+    for a in assignments:
+        if a.session:
+            a.session.historic_group = None
+
+    groups = [a.session.group for a in assignments if a.session and a.session.group]
+    group_replacements = find_history_replacements_active_at(groups, meeting_time)
+
+    for a in assignments:
+        if a.session and a.session.group:
+            a.session.historic_group = group_replacements.get(a.session.group_id)
+
+    # replace group parents with historic counterparts
+    for a in assignments:
+        if a.session and a.session.historic_group:
+            a.session.historic_group.historic_parent = None
+
+    parents = Group.objects.filter(pk__in=set(a.session.historic_group.parent_id for a in assignments if a.session and a.session.historic_group and a.session.historic_group.parent_id))
+    parent_replacements = find_history_replacements_active_at(parents, meeting_time)
+
+    for a in assignments:
+        if a.session and a.session.historic_group and a.session.historic_group.parent_id:
+            a.session.historic_group.historic_parent = parent_replacements.get(a.session.historic_group.parent_id)
+
+    return assignments
+
+def read_agenda_file(num, doc):
+    # XXXX FIXME: the path fragment in the code below should be moved to
+    # settings.py.  The *_PATH settings should be generalized to format()
+    # style python format, something like this:
+    #  DOC_PATH_FORMAT = { "agenda": "/foo/bar/agenda-{meeting.number}/agenda-{meeting-number}-{doc.group}*", }
+    path = os.path.join(settings.AGENDA_PATH, "%s/agenda/%s" % (num, doc.external_url))
+    if os.path.exists(path):
+        with open(path) as f:
+            return f.read()
+    else:
+        return None
+
+def convert_draft_to_pdf(doc_name):
+    inpath = os.path.join(settings.IDSUBMIT_REPOSITORY_PATH, doc_name + ".txt")
+    outpath = os.path.join(settings.INTERNET_DRAFT_PDF_PATH, doc_name + ".pdf")
+
+    try:
+        infile = open(inpath, "r")
+    except IOError:
+        return
+
+    t,tempname = mkstemp()
+    os.close(t)
+    tempfile = open(tempname, "w")
+
+    pageend = 0;
+    newpage = 0;
+    formfeed = 0;
+    for line in infile:
+        line = re.sub("\r","",line)
+        line = re.sub("[ \t]+$","",line)
+        if re.search("\[?[Pp]age [0-9ivx]+\]?[ \t]*$",line):
+            pageend=1
+            tempfile.write(line)
+            continue
+        if re.search("^[ \t]*\f",line):
+            formfeed=1
+            tempfile.write(line)
+            continue
+        if re.search("^ *INTERNET.DRAFT.+[0-9]+ *$",line) or re.search("^ *Internet.Draft.+[0-9]+ *$",line) or re.search("^draft-[-a-z0-9_.]+.*[0-9][0-9][0-9][0-9]$",line) or re.search("^RFC.+[0-9]+$",line):
+            newpage=1
+        if re.search("^[ \t]*$",line) and pageend and not newpage:
+            continue
+        if pageend and newpage and not formfeed:
+            tempfile.write("\f")
+        pageend=0
+        formfeed=0
+        newpage=0
+        tempfile.write(line)
+
+    infile.close()
+    tempfile.close()
+    t,psname = mkstemp()
+    os.close(t)
+    pipe("enscript --margins 76::76: -B -q -p "+psname + " " +tempname)
+    os.unlink(tempname)
+    pipe("ps2pdf "+psname+" "+outpath)
+    os.unlink(psname)
+
 def agenda_permissions(meeting, schedule, user):
     # do this in positive logic.
     cansee = False

ietf/meeting/models.py

@@ -638,16 +638,6 @@ def sessions_that_can_meet(self):
             self._cached_sessions_that_can_meet = self.meeting.sessions_that_can_meet.all()
         return self._cached_sessions_that_can_meet
 
-    def area_list(self):
-        return ( self.assignments.filter(session__group__type__slug__in=['wg', 'rg', 'ag', 'iab'],
-                                         session__group__parent__isnull=False)
-                 .order_by('session__group__parent__acronym')
-                 .values_list('session__group__parent__acronym', flat=True)
-                 .distinct() )
-
-    def groups(self):
-        return Group.objects.filter(type__slug__in=['wg', 'rg', 'ag', 'iab'], parent__isnull=False, session__scheduledsession__schedule=self).exclude(session__scheduledsession__timeslot__type__in=['lead','offagenda']).distinct().order_by('parent__acronym', 'acronym')
-
     # calculate badness of entire schedule
     def calc_badness(self):
         # now calculate badness
@@ -768,6 +758,35 @@ def json_dict(self, host_scheme):
         ss["pinned"]   = self.pinned
         return ss
 
+    def slug(self):
+        """Return sensible id string for session, e.g. suitable for use as HTML anchor."""
+        components = []
+
+        if not self.timeslot:
+            components.append("unknown")
+
+        if not self.session or not (getattr(self.session, "historic_group") or self.session.group):
+            components.append("unknown")
+        else:
+            components.append(self.timeslot.time.strftime("%a-%H%M"))
+
+            g = getattr(self.session, "historic_group", None) or self.session.group
+
+            if self.timeslot.type_id in ('break', 'reg', 'other'):
+                components.append(g.acronym)
+
+            if self.timeslot.type_id in ('session', 'plenary'):
+                if self.timeslot.type_id == "plenary":
+                    components.append("1plenary")
+                else:
+                    p = getattr(g, "historic_parent", None) or g.parent
+                    if p and p.type_id in ("area", "irtf"):
+                        components.append(p.acronym)
+
+                components.append(g.acronym)
+
+        return u"-".join(components).lower()
+
 class Constraint(models.Model):
     """
     Specifies a constraint on the scheduling.
@@ -886,14 +905,30 @@ class Session(models.Model):
 
     # Should work on how materials are captured so that deleted things are no longer associated with the session
     # (We can keep the information about something being added to and removed from a session in the document's history)
+    def get_material(self, material_type, only_one):
+        if hasattr(self, "prefetched_active_materials"):
+            l = [d for d in self.prefetched_active_materials if d.type_id == material_type]
+            for d in l:
+                d.meeting_related = lambda: True
+        else:
+            l = self.materials.filter(type=material_type).exclude(states__type=material_type, states__slug='deleted').order_by("order")
+
+        if only_one:
+            if l:
+                return l[0]
+            else:
+                return None
+        else:
+            return l
+
     def agenda(self):
-        return self.materials.filter(type='agenda').exclude(states__type='agenda',states__slug='deleted').first()
+        return self.get_material("agenda", only_one=True)
 
     def minutes(self):
-        return self.materials.filter(type='minutes').exclude(states__type='minutes',states__slug='deleted').first()
+        return self.get_material("minutes", only_one=True)
 
     def slides(self):
-        return list(self.materials.filter(type='slides').exclude(states__type='slides',states__slug='deleted').order_by('order'))
+        return list(self.get_material("slides", only_one=False))
 
     def __unicode__(self):
         if self.meeting.type_id == "interim":
@@ -908,9 +943,6 @@ def __unicode__(self):
                 ss0name = ','.join([x.timeslot.time.strftime("%a-%H%M") for x in ss])
         return u"%s: %s %s %s" % (self.meeting, self.group.acronym, self.name, ss0name)
 
-    def is_bof(self):
-        return self.group.is_bof();
-
     @property
     def short_name(self):
         if self.name:
@@ -994,7 +1026,7 @@ def json_dict(self, host_scheme):
         sess1['name']           = str(self.name)
         sess1['title']          = str(self.short_name)
         sess1['short_name']     = str(self.short_name)
-        sess1['bof']            = str(self.is_bof())
+        sess1['bof']            = str(self.group.is_bof())
         sess1['agenda_note']    = str(self.agenda_note)
         sess1['attendees']      = str(self.attendees)
         sess1['status']         = str(self.status)
@@ -1024,18 +1056,6 @@ def agenda_text(self):
         else:
             return "The agenda has not been uploaded yet."
 
-    # FIXME - This used to be called 'type'. It is only used in agenda.csv and agenda.txt.
-    # It will return the _wrong thing_ if you look back at an agenda of an earlier meeting
-    # where group X was a BOF at the time, but is now a WG.
-    # It also doesn't return anything useful for RG, area sessions, or anything that's not group type 'wg'.
-    # A better thing to do is find a way to note when a meeting was a BoF meeting and use that, removing this
-    # function altogether.
-    def lame_description(self):
-        if self.group.type.slug in [ "wg" ]:
-            return "BOF" if self.group.state.slug in ["bof", "bof-conc"] else "WG"
-        else:
-            return ""
-
     def ical_status(self):
         if self.status.slug == 'canceled': # sic
             return "CANCELLED"
@@ -1050,13 +1070,13 @@ def agenda_file(self):
         if not hasattr(self, '_agenda_file'):
             self._agenda_file = ""
 
-            docs = self.materials.filter(type="agenda", states__type="agenda", states__slug="active")
-            if not docs:
+            agenda = self.agenda()
+            if not agenda:
                 return ""
 
             # we use external_url at the moment, should probably regularize
             # the filenames to match the document name instead
-            filename = docs[0].external_url
+            filename = agenda.external_url
             self._agenda_file = "%s/agenda/%s" % (self.meeting.number, filename)
 
         return self._agenda_file