Another round of iesg page optimizations. Reduced additional instances of individual lookups to a single lookup followed by annotation of document objects. Reduced several instances of repeating the same work multiple times (queries for telechat documents, queries for telechat dates).

 - Legacy-Id: 15010

Author: levkowetz

ietf/doc/utils_search.py

@@ -4,20 +4,34 @@
 import debug                            # pyflakes:ignore
 
 from ietf.community.utils import augment_docs_with_tracking_info
-from ietf.doc.models import Document, DocAlias, RelatedDocument, DocEvent, TelechatDocEvent
+from ietf.doc.models import Document, DocAlias, RelatedDocument, DocEvent, TelechatDocEvent, BallotDocEvent
 from ietf.doc.expire import expirable_draft
-from ietf.meeting.models import SessionPresentation
+
 
 def wrap_value(v):
     return lambda: v
 
 
-def fill_in_document_table_attributes(docs):
+def fill_in_telechat_date(docs, doc_dict=None, doc_ids=None):
+    if doc_dict is None:
+        doc_dict = dict((d.pk, d) for d in docs)
+        doc_ids = doc_dict.keys()
+    if doc_ids is None:
+        doc_ids = doc_dict.keys()        
+
+    seen = set()
+    for e in TelechatDocEvent.objects.filter(doc__in=doc_ids, type="scheduled_for_telechat").order_by('-time'):
+        if e.doc_id not in seen:
+            d = doc_dict[e.doc_id]
+            d.telechat_date = wrap_value(d.telechat_date(e))
+            seen.add(e.doc_id)
+
+def fill_in_document_table_attributes(docs, have_telechat_date=False):
     # fill in some attributes for the document table results to save
     # some hairy template code and avoid repeated SQL queries
 
-    docs_dict = dict((d.pk, d) for d in docs)
-    doc_ids = docs_dict.keys()
+    doc_dict = dict((d.pk, d) for d in docs)
+    doc_ids = doc_dict.keys()
 
     rfc_aliases = dict(DocAlias.objects.filter(name__startswith="rfc", document__in=doc_ids).values_list("document", "name"))
 
@@ -32,22 +46,26 @@ def fill_in_document_table_attributes(docs):
             d.latest_event_cache[e] = None
 
     for e in DocEvent.objects.filter(doc__in=doc_ids, type__in=event_types).order_by('time'):
-        docs_dict[e.doc_id].latest_event_cache[e.type] = e
+        doc_dict[e.doc_id].latest_event_cache[e.type] = e
 
-    # telechat date, can't do this with above query as we need to get TelechatDocEvents out
     seen = set()
-    for e in TelechatDocEvent.objects.filter(doc__in=doc_ids, type="scheduled_for_telechat").order_by('-time'):
-        if e.doc_id not in seen:
-            d = docs_dict[e.doc_id]
-            d.telechat_date = wrap_value(d.telechat_date(e))
+    for e in BallotDocEvent.objects.filter(doc__in=doc_ids, type__in=('created_ballot', 'closed_ballot')).order_by('-time'):
+        if not e.doc_id in seen:
+            doc_dict[e.doc_id].ballot = e if e.type == 'created_ballot' else None
             seen.add(e.doc_id)
 
+    if not have_telechat_date:
+        fill_in_telechat_date(docs, doc_dict, doc_ids)
+
     # on agenda in upcoming meetings
-    presentations = SessionPresentation.objects.filter(session__meeting__date__gte=datetime.date.today()-datetime.timedelta(days=7)).select_related('session', 'document')
-    session_list = [ (p.document, p.session) for p in presentations ]
-    sessions = dict( (d, []) for (d, s) in session_list )
-    for (d, s) in session_list:
-        sessions[d].append(s)
+    sessions = {}
+#    debug.mark()
+#    presentations = SessionPresentation.objects.filter(session__meeting__date__gte=datetime.date.today()-datetime.timedelta(days=7)).select_related('session', 'document')
+#   session_list = [ (p.document, p.session) for p in presentations ]
+#    sessions = dict( (d, []) for (d, s) in session_list )
+#    for (d, s) in session_list:
+#        sessions[d].append(s)
+#    debug.clock('presentations')
 
     # misc
     for d in docs:
@@ -79,6 +97,9 @@ def fill_in_document_table_attributes(docs):
 
         d.sessions = sessions[d] if d in sessions else []
 
+        e = d.latest_event_cache.get('started_iesg_proces', None)
+        d.balloting_started = e.time if e else datetime.datetime.min
+
     # RFCs
 
     # errata
@@ -88,7 +109,7 @@ def fill_in_document_table_attributes(docs):
 
     # obsoleted/updated by
     for a in rfc_aliases:
-        d = docs_dict[a]
+        d = doc_dict[a]
         d.obsoleted_by_list = []
         d.updated_by_list = []
 
@@ -97,7 +118,7 @@ def fill_in_document_table_attributes(docs):
     rel_rfc_aliases = dict(DocAlias.objects.filter(name__startswith="rfc",
                                                    document__in=[rel.source_id for rel in xed_by]).values_list('document', 'name'))
     for rel in xed_by:
-        d = docs_dict[rel.target.document_id]
+        d = doc_dict[rel.target.document_id]
         if rel.relationship_id == "obs":
             l = d.obsoleted_by_list
         elif rel.relationship_id == "updates":

ietf/group/views.py

@@ -382,6 +382,7 @@ def chartering_groups(request):
 
         for g in t.chartering_groups:
             g.chartering_type = get_chartering_type(g.charter)
+            g.charter.ballot = g.charter.active_ballot()
 
     return render(request, 'group/chartering_groups.html',
                   dict(charter_states=charter_states,

ietf/iesg/agenda.py

@@ -7,7 +7,8 @@
 from django.conf import settings
 from django.http import Http404
 
-from ietf.doc.models import Document, TelechatDocEvent, LastCallDocEvent, ConsensusDocEvent
+from ietf.doc.models import Document, LastCallDocEvent, ConsensusDocEvent
+from ietf.doc.utils_search import fill_in_telechat_date
 from ietf.iesg.models import TelechatDate, TelechatAgendaItem
 from ietf.review.utils import review_requests_to_list_for_docs
 
@@ -147,19 +148,21 @@ def fill_in_agenda_administrivia(date, sections):
 
         sections[s]["text"] = t
 
-def fill_in_agenda_docs(date, sections, matches=None):
-    if not matches:
-        matches = Document.objects.filter(docevent__telechatdocevent__telechat_date=date)
-        matches = matches.select_related("stream", "group").distinct()
+def fill_in_agenda_docs(date, sections, docs=None):
+    if not docs:
+        docs = Document.objects.filter(docevent__telechatdocevent__telechat_date=date)
+        docs = docs.select_related("stream", "group").distinct()
+        fill_in_telechat_date(docs)
 
-    review_requests_for_docs = review_requests_to_list_for_docs(matches)
+    review_requests_for_docs = review_requests_to_list_for_docs(docs)
 
-    for doc in matches:
-        if doc.latest_event(TelechatDocEvent, type="scheduled_for_telechat").telechat_date != date:
+    for doc in docs:
+        if doc.telechat_date() != date:
             continue
 
-        e = doc.latest_event(type="started_iesg_process")
-        doc.balloting_started = e.time if e else datetime.datetime.min
+        if not hasattr(doc, 'balloting_started'):
+            e = doc.latest_event(type="started_iesg_process")
+            doc.balloting_started = e.time if e else datetime.datetime.min
 
         if doc.type_id == "draft":
             s = doc.get_state("draft-iana-review")

ietf/iesg/utils.py

@@ -2,9 +2,11 @@
 
 import debug                            # pyflakes:ignore
 
-from ietf.doc.models import Document, TelechatDocEvent, STATUSCHANGE_RELATIONS
+from ietf.doc.models import Document, STATUSCHANGE_RELATIONS
+from ietf.doc.utils_search import fill_in_telechat_date
 from ietf.iesg.agenda import get_doc_section
 
+
 TelechatPageCount = namedtuple('TelechatPageCount',['for_approval','for_action','related'])
 
 def telechat_page_count(date=None, docs=None):
@@ -13,7 +15,8 @@ def telechat_page_count(date=None, docs=None):
 
     if not docs:
         candidates = Document.objects.filter(docevent__telechatdocevent__telechat_date=date).distinct() 
-        docs = [ doc for doc in candidates if doc.latest_event(TelechatDocEvent,type='scheduled_for_telechat').telechat_date==date ]
+        fill_in_telechat_date(candidates)
+        docs = [ doc for doc in candidates if doc.telechat_date()==date ]
 
     for_action =[d for d in docs if get_doc_section(d).endswith('.3')]
 

ietf/iesg/views.py

@@ -49,17 +49,18 @@
 from django.http import HttpResponse
 from django.shortcuts import render, redirect
 from django.contrib.sites.models import Site
+#from django.views.decorators.cache import cache_page
+#from django.views.decorators.vary import vary_on_cookie
 
-
-from ietf.doc.models import Document, State, TelechatDocEvent, LastCallDocEvent, ConsensusDocEvent, DocEvent, IESG_BALLOT_ACTIVE_STATES
+from ietf.doc.models import Document, State, LastCallDocEvent, ConsensusDocEvent, DocEvent, IESG_BALLOT_ACTIVE_STATES
 from ietf.doc.utils import update_telechat, augment_events_with_revision
 from ietf.group.models import GroupMilestone, Role
 from ietf.iesg.agenda import agenda_data, agenda_sections, fill_in_agenda_docs, get_agenda_date
 from ietf.iesg.models import TelechatDate
 from ietf.iesg.utils import telechat_page_count
 from ietf.ietfauth.utils import has_role, role_required, user_is_person
 from ietf.person.models import Person
-from ietf.doc.utils_search import fill_in_document_table_attributes
+from ietf.doc.utils_search import fill_in_document_table_attributes, fill_in_telechat_date
 
 def review_decisions(request, year=None):
     events = DocEvent.objects.filter(type__in=("iesg_disapproved", "iesg_approved"))
@@ -288,8 +289,11 @@ def agenda_package(request, date=None):
 def agenda_documents_txt(request):
     dates = list(TelechatDate.objects.active().order_by('date').values_list("date", flat=True)[:4])
 
+    all_docs = Document.objects.filter(docevent__telechatdocevent__telechat_date__in=dates).distinct()
     docs = []
-    for d in Document.objects.filter(docevent__telechatdocevent__telechat_date__in=dates).distinct():
+    fill_in_telechat_date(all_docs)
+
+    for d in all_docs:
         date = d.telechat_date()
         if date in dates:
             d.computed_telechat_date = date
@@ -358,11 +362,13 @@ def agenda_documents(request):
     dates = list(TelechatDate.objects.active().order_by('date').values_list("date", flat=True)[:4])
 
     docs_by_date = dict((d, []) for d in dates)
-    for doc in (Document.objects
-                                .filter(docevent__telechatdocevent__telechat_date__in=dates)
-                                .select_related('stream', 'group', 'intended_std_level')
-                                .distinct()):
-        d = doc.latest_event(TelechatDocEvent, type="scheduled_for_telechat").telechat_date
+    docs = Document.objects.filter(docevent__telechatdocevent__telechat_date__in=dates).distinct()
+    docs = docs.select_related("ad", "std_level", "intended_std_level", "group", "stream", "shepherd", )
+    # No prefetch-related -- turns out not to be worth it
+
+    fill_in_telechat_date(docs)
+    for doc in docs:
+        d = doc.telechat_date()
         if d in docs_by_date:
             docs_by_date[d].append(doc)
 
@@ -380,13 +386,13 @@ def agenda_documents(request):
         sections = agenda_sections()
         # augment the docs with the search attributes, since we're using
         # the search_result_row view to display them (which expects them)
-        fill_in_document_table_attributes(docs_by_date[date])
+        fill_in_document_table_attributes(docs_by_date[date], have_telechat_date=True)
         fill_in_agenda_docs(date, sections, docs_by_date[date])
         pages = telechat_page_count(docs=docs_by_date[date]).for_approval
 
         telechats.append({
-                "date":date,
-                "pages":pages,
+                "date":     date,
+                "pages":    pages,
                 "sections": sorted((num, section) for num, section in sections.iteritems()
                                    if "2" <= num < "5")
                 })
@@ -417,6 +423,7 @@ def past_documents(request):
         doc.milestones = doc.groupmilestone_set.filter(state="active").order_by("time").select_related("group")
         doc.blocking_positions = blocking_positions
         doc.telechat = doc.previous_telechat_date()
+        doc.ballot = ballot
 
         if doc.telechat:
             docs.append(doc)
@@ -431,9 +438,11 @@ def past_documents(request):
 def telechat_docs_tarfile(request, date):
     date = get_agenda_date(date)
 
+    all_docs = Document.objects.filter(docevent__telechatdocevent__telechat_date=date).distinct()
+    fill_in_telechat_date(all_docs)
     docs = []
-    for d in Document.objects.filter(docevent__telechatdocevent__telechat_date=date).distinct():
-        if d.latest_event(TelechatDocEvent, type="scheduled_for_telechat").telechat_date == date:
+    for d in all_docs:
+        if d.telechat_date() == date:
             docs.append(d)
 
     response = HttpResponse(content_type='application/octet-stream')
@@ -491,6 +500,7 @@ def discusses(request):
         doc.for_me = user_is_person(request.user, doc.ad)
         doc.milestones = doc.groupmilestone_set.filter(state="active").order_by("time").select_related("group")
         doc.blocking_positions = blocking_positions
+        doc.ballot = ballot
 
         docs.append(doc)
 

ietf/templates/doc/search/status_columns.html

@@ -2,8 +2,8 @@
 {% load ietf_filters ballot_icon %}
 
 <td class="status">
-  <div class="pull-right">
-    {% ballot_icon doc %}
+  <div class="pull-right" id="ballot-icon-{{doc.name}}">
+    {% if doc.ballot %}{% ballot_icon doc %}{% endif %}
   </div>
 
   {% if not doc.get_state_slug == "rfc" %}