Merged [4772] from rjsparks@nostrum.com:

Liasion Manager:
* Patch from Yaco to avoid resetting the From field when changing other items on the form
* Gave the secretariat the ability to find/approve any unapproved liaisons
* Changed all the email code to use ietf.mail.utils (and removed the fake-mail concept)

Charter documents and the Agenda pages:
* Added charter documents to iesg/agenda and iesg/agenda/documents
* Synced the ordering of drafts on iesg/agenda and iesg/agenda/documents
* Allow setting a responsible AD for charter documents
* Changed the UI of the charter page to use editlink for changing attributes and buttons for actions (to align with drafts and conflict-reviews)

Moderator package:
* Refactor: Simplified access to the current BallotDocEvent from a Document
* Added functions to BallotDocEvents? to faciliate access to BallotPositionDocEvents?, both for all positions, and current AD postions.
* Updated the moderator package to use the Documents from _agenda_data.
* Added a filter to assist with rendering the moderator package.
* Fixed a bug where different functions in idrfc/views_ballot were using log_state_changed expecting different implementations (a cleanup task should reconcile the _three_ implementations in the codebase of that function).

Cleanup from codesprint:
* Removed some duplication between doc/util and doc/models by moving things into doc/models
* Do not show non-empty discuss text when the ballot position is not blocking
* Added a migration to update non-blocking ballot positions that have non-empty discuss text


DEPLOYMENT NOTES
Please be aware that migration step will take a few minutes to complete.

Fixes bug 865
 - Legacy-Id: 4780
Note: SVN reference [4772] has been migrated to Git commit e5c3a5a

Author: levkowetz

ietf/doc/migrations/0004_clean_stale_discusses.py

@@ -261,29 +261,21 @@ def active_defer_event(self):
             return self.latest_event(type="changed_document", desc__startswith="State changed to <b>IESG Evaluation - Defer</b>")
         return None
 
+# This, and several other ballot related functions here, assume that there is only one active ballot for a document at any point in time.
+# If that assumption is violated, they will only expose the most recently created ballot
+    def ballot_open(self, ballot_type_slug):
+        e = self.latest_event(BallotDocEvent, ballot_type__slug=ballot_type_slug)
+        return e and not e.type == "closed_ballot"
+
     def active_ballot(self):
+        """Returns the most recently created ballot if it isn't closed."""
         ballot = self.latest_event(BallotDocEvent, type="created_ballot")
-        e = self.latest_event(BallotDocEvent, ballot_type__slug=ballot.ballot_type.slug) if ballot else None
-        open = e and not e.type == "closed_ballot"
-        return ballot.ballot_type if open else None
+        open = self.ballot_open(ballot.ballot_type.slug) if ballot else False
+        return ballot if open else None
 
-    def active_ballot_positions(self):
-        """Return dict mapping each active AD to a current ballot position (or None if they haven't voted)."""
-        active_ads = list(Person.objects.filter(role__name="ad", role__group__state="active"))
-        res = {}
-    
-        ballot = self.latest_event(BallotDocEvent, type="created_ballot")
-        positions = BallotPositionDocEvent.objects.filter(doc=self, type="changed_ballot_position", ad__in=active_ads, ballot=ballot).select_related('ad', 'pos').order_by("-time", "-id")
-   
-        for pos in positions:
-            if pos.ad not in res:
-                res[pos.ad] = pos
-    
-        for ad in active_ads:
-            if ad not in res:
-                res[ad] = None
-    
-        return res.values()
+    def most_recent_ietflc(self):
+        """Returns the most recent IETF LastCallDocEvent for this document"""
+        return self.latest_event(LastCallDocEvent,type="sent_last_call")
 
     def displayname_with_link(self):
         return '<a href="%s">%s-%s</a>' % (self.get_absolute_url(), self.name , self.rev)
@@ -528,6 +520,59 @@ class Meta:
 class BallotDocEvent(DocEvent):
     ballot_type = models.ForeignKey(BallotType)
 
+    def active_ad_positions(self):
+        """Return dict mapping each active AD to a current ballot position (or None if they haven't voted)."""
+        active_ads = list(Person.objects.filter(role__name="ad", role__group__state="active"))
+        res = {}
+    
+        if self.doc.latest_event(BallotDocEvent, type="created_ballot") == self:
+        
+            positions = BallotPositionDocEvent.objects.filter(type="changed_ballot_position",ad__in=active_ads, ballot=self).select_related('ad', 'pos').order_by("-time", "-id")
+   
+            for pos in positions:
+                if pos.ad not in res:
+                    res[pos.ad] = pos
+    
+            for ad in active_ads:
+                if ad not in res:
+                    res[ad] = None
+        return res
+
+    def all_positions(self):
+        """Return array holding the current and past positions per AD"""
+
+        positions = []
+        seen = {}
+        active_ads = list(Person.objects.filter(role__name="ad", role__group__state="active").distinct())
+        for e in BallotPositionDocEvent.objects.filter(type="changed_ballot_position", ballot=self).select_related('ad', 'pos').order_by("-time", '-id'):
+            if e.ad not in seen:
+                e.old_ad = e.ad not in active_ads
+                e.old_positions = []
+                positions.append(e)
+                seen[e.ad] = e
+            else:
+                latest = seen[e.ad]
+                if latest.old_positions:
+                    prev = latest.old_positions[-1]
+                else:
+                    prev = latest.pos
+    
+                if e.pos != prev:
+                    latest.old_positions.append(e.pos)
+    
+        # add any missing ADs through fake No Record events
+        norecord = BallotPositionName.objects.get(slug="norecord")
+        for ad in active_ads:
+            if ad not in seen:
+                e = BallotPositionDocEvent(type="changed_ballot_position", doc=self.doc, ad=ad)
+                e.pos = norecord
+                e.old_ad = False
+                e.old_positions = []
+                positions.append(e)
+
+        positions.sort(key=lambda p: (p.old_ad, p.ad.last_name()))
+        return positions
+
 class BallotPositionDocEvent(DocEvent):
     ballot = models.ForeignKey(BallotDocEvent, null=True, default=None) # default=None is a temporary migration period fix, should be removed when charter branch is live
     ad = models.ForeignKey(Person)

ietf/doc/models.py

@@ -550,10 +550,9 @@ def ballot_issued(self):
     def active_positions(self):
         """Returns a list of dicts, with AD and Position tuples"""
         from ietf.person.proxy import IESGLogin as IESGLoginProxy
-        from ietf.doc.utils import active_ballot_positions
 
         res = []
-        for ad, pos in active_ballot_positions(self).iteritems():
+        for ad, pos in self.active_ballot().active_ad_positions().iteritems():
             res.append(dict(ad=IESGLoginProxy().from_object(ad), pos=Position().from_object(pos) if pos else None))
 
         res.sort(key=lambda x: x["ad"].last_name)

ietf/doc/proxy.py

@@ -13,7 +13,7 @@
 from ietf.utils.test_utils import login_testing_unauthorized
 from ietf.utils.test_data  import make_test_data
 from ietf.utils.mail import outbox
-from ietf.doc.utils import active_ballot, create_ballot_if_not_open, ballot_open
+from ietf.doc.utils import create_ballot_if_not_open
 from ietf.doc.views_conflict_review import default_approval_text
 
 from ietf.doc.models import Document,DocEvent,NewRevisionDocEvent,BallotPositionDocEvent,TelechatDocEvent,DocAlias,State
@@ -105,7 +105,7 @@ def test_change_state(self):
         review_doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
         self.assertEquals(review_doc.get_state('conflrev').slug,'adrev')
         self.assertTrue(review_doc.latest_event(DocEvent,type="added_comment").desc.startswith('RDNK84ZD'))
-        self.assertFalse(active_ballot(review_doc))
+        self.assertFalse(review_doc.active_ballot())
 
         # successful change to IESG Evaluation 
         iesgeval_pk = str(State.objects.get(slug='iesgeval',type__slug='conflrev').pk)
@@ -114,7 +114,7 @@ def test_change_state(self):
         review_doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
         self.assertEquals(review_doc.get_state('conflrev').slug,'iesgeval')
         self.assertTrue(review_doc.latest_event(DocEvent,type="added_comment").desc.startswith('TGmZtEjt'))
-        self.assertTrue(active_ballot(review_doc))
+        self.assertTrue(review_doc.active_ballot())
         self.assertEquals(review_doc.latest_event(BallotPositionDocEvent, type="changed_ballot_position").pos_id,'yes')
 
 
@@ -234,7 +234,7 @@ def approve_test_helper(self,approve_type):
 
         doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
         self.assertEquals(doc.get_state_slug(),approve_type+'-sent')
-        self.assertFalse(ballot_open(doc, "conflrev"))
+        self.assertFalse(doc.ballot_open("conflrev"))
 
         self.assertEquals(len(outbox), messages_before + 1)
         self.assertTrue('Results of IETF-conflict review' in outbox[-1]['Subject'])

ietf/doc/tests_conflict_review.py

@@ -36,34 +36,6 @@ def get_tags_for_stream_id(stream_id):
     else:
         return []
 
-# This, and several other utilities here, assume that there is only one active ballot for a document at any point in time.
-# If that assumption is violated, they will only expose the most recently created ballot
-def active_ballot(doc):
-    """Returns the most recently created ballot if it isn't closed."""
-    ballot = doc.latest_event(BallotDocEvent, type="created_ballot")
-    open = ballot_open(doc,ballot.ballot_type.slug) if ballot else False
-    return ballot if open else None
-     
-
-def active_ballot_positions(doc, ballot=None):
-    """Return dict mapping each active AD to a current ballot position (or None if they haven't voted)."""
-    active_ads = list(Person.objects.filter(role__name="ad", role__group__state="active"))
-    res = {}
-
-    if not ballot:
-        ballot = doc.latest_event(BallotDocEvent, type="created_ballot")
-    positions = BallotPositionDocEvent.objects.filter(doc=doc, type="changed_ballot_position", ad__in=active_ads, ballot=ballot).select_related('ad', 'pos').order_by("-time", "-id")
-
-    for pos in positions:
-        if pos.ad not in res:
-            res[pos.ad] = pos
-
-    for ad in active_ads:
-        if ad not in res:
-            res[ad] = None
-
-    return res
-
 def needed_ballot_positions(doc, active_positions):
     '''Returns text answering the question "what does this document
     need to pass?".  The return value is only useful if the document
@@ -102,20 +74,16 @@ def needed_ballot_positions(doc, active_positions):
 
     return " ".join(answer)
 
-def ballot_open(doc, ballot_type_slug):
-    e = doc.latest_event(BallotDocEvent, ballot_type__slug=ballot_type_slug)
-    return e and not e.type == "closed_ballot"
-
 def create_ballot_if_not_open(doc, by, ballot_type_slug):
-    if not ballot_open(doc, ballot_type_slug):
+    if not doc.ballot_open(ballot_type_slug):
         e = BallotDocEvent(type="created_ballot", by=by, doc=doc)
         e.ballot_type = BallotType.objects.get(doc_type=doc.type, slug=ballot_type_slug)
         e.desc = u'Created "%s" ballot' % e.ballot_type.name
         e.save()
 
 def close_open_ballots(doc, by):
     for t in BallotType.objects.filter(doc_type=doc.type_id):
-        if ballot_open(doc, t.slug):
+        if doc.ballot_open(t.slug):
             e = BallotDocEvent(type="closed_ballot", doc=doc, by=by)
             e.ballot_type = t
             e.desc = 'Closed "%s" ballot' % t.name

ietf/doc/utils.py

@@ -37,7 +37,6 @@
 from ietf.idrfc.idrfc_wrapper import position_to_string, BALLOT_ACTIVE_STATES
 from ietf.idtracker.templatetags.ietf_filters import in_group, timesince_days
 from ietf.ietfauth.decorators import has_role
-from ietf.doc.utils import active_ballot_positions
 from ietf.doc.models import BallotDocEvent
 
 register = template.Library()
@@ -83,7 +82,7 @@ def sort_key(t):
         else:
             return (1, pos.pos.order)
 
-    positions = list(active_ballot_positions(doc, ballot).items())
+    positions = list(doc.active_ballot().active_ad_positions().items())
     positions.sort(key=sort_key)
 
     cm = ""

ietf/idrfc/templatetags/ballot_icon.py

@@ -38,7 +38,6 @@
 from ietf.idrfc.idrfc_wrapper import position_to_string, BALLOT_ACTIVE_STATES
 from ietf.idtracker.templatetags.ietf_filters import in_group, timesince_days
 from ietf.ietfauth.decorators import has_role
-from ietf.doc.utils import active_ballot_positions, active_ballot
 from ietf.doc.models import BallotDocEvent, BallotPositionDocEvent
 
 from datetime import date
@@ -90,7 +89,7 @@ def sort_key(t):
         else:
             return (1, pos.pos.order)
 
-    positions = list(active_ballot_positions(doc, ballot).items())
+    positions = list(doc.active_ballot().active_ad_positions().items())
     positions.sort(key=sort_key)
 
     cm = ""
@@ -144,7 +143,7 @@ def my_position(doc, user):
         return None
     if not in_group(user, "Area_Director"):
         return None
-    ballot = active_ballot(doc)
+    ballot = doc.active_ballot()
     pos = "No Record"
     if ballot:
       changed_pos = doc.latest_event(BallotPositionDocEvent, type="changed_ballot_position", ad__name=user_name, ballot=ballot)

ietf/idrfc/templatetags/ballot_icon_redesign.py

@@ -35,7 +35,8 @@
 from ietf.message.utils import infer_message
 from ietf.person.models import Person
 
-from ietf.doc.utils import log_state_changed
+from ietf.doc.utils import log_state_changed as docutil_log_state_changed
+from ietf.idrfc.utils import log_state_changed as idrfcutil_log_state_changed
 
 BALLOT_CHOICES = (("yes", "Yes"),
                   ("noobj", "No Objection"),
@@ -431,7 +432,7 @@ def defer_ballotREDESIGN(request, name):
         elif doc.type_id == 'conflrev':
             doc.set_state(State.objects.get(type='conflrev', slug='defer'))
 
-        e = log_state_changed(request, doc, login, doc.friendly_state(), prev_state)
+        e = docutil_log_state_changed(request, doc, login, doc.friendly_state(), prev_state)
 
         doc.time = e.time
         doc.save()
@@ -481,7 +482,7 @@ def undefer_ballotREDESIGN(request, name):
         elif doc.type_id == 'conflrev':
             doc.set_state(State.objects.get(type='conflrev',slug='iesgeval'))
 
-        e = log_state_changed(request, doc, login, doc.friendly_state(), prev_state)
+        e = docutil_log_state_changed(request, doc, login, doc.friendly_state(), prev_state)
 
         doc.time = e.time
         doc.save()
@@ -557,7 +558,7 @@ def lastcalltext(request, name):
                 if "send_last_call_request" in request.POST:
                     doc.idinternal.change_state(IDState.objects.get(document_state_id=IDState.LAST_CALL_REQUESTED), None)
 
-                    change = log_state_changed(request, doc, login)
+                    change = idrfcutil_log_state_changed(request, doc, login)
                     email_owner(request, doc, doc.idinternal.job_owner, login, change)
                     request_last_call(request, doc)
 
@@ -646,7 +647,7 @@ def lastcalltextREDESIGN(request, name):
                     if prev_tag:
                         doc.tags.remove(prev_tag)
 
-                    e = log_state_changed(request, doc, login, prev, prev_tag)
+                    e = idrfcutil_log_state_changed(request, doc, login, prev, prev_tag)
 
                     doc.time = e.time
                     doc.save()
@@ -1093,7 +1094,7 @@ def approve_ballotREDESIGN(request, name):
 
         change_description = e.desc + " and state has been changed to %s" % doc.get_state("draft-iesg").name
 
-        e = log_state_changed(request, doc, login, prev, prev_tag)
+        e = idrfcutil_log_state_changed(request, doc, login, prev, prev_tag)
 
         doc.time = e.time
         doc.save()
@@ -1153,7 +1154,7 @@ def make_last_call(request, name):
             doc.idinternal.event_date = date.today()
             doc.idinternal.save()
 
-            log_state_changed(request, doc, login)
+            idrfcutil_log_state_changed(request, doc, login)
 
             doc.lc_sent_date = form.cleaned_data['last_call_sent_date']
             doc.lc_expiration_date = form.cleaned_data['last_call_expiration_date']
@@ -1216,7 +1217,7 @@ def make_last_callREDESIGN(request, name):
             if prev_tag:
                 doc.tags.remove(prev_tag)
 
-            e = log_state_changed(request, doc, login, prev, prev_tag)
+            e = idrfcutil_log_state_changed(request, doc, login, prev, prev_tag)
 
             doc.time = e.time
             doc.save()

ietf/idrfc/views_ballot.py

@@ -143,7 +143,7 @@ def document_main(request, name, rev=None):
 
         ballot_summary = None
         if doc.get_state_slug() in ("intrev", "iesgrev"):
-            ballot_summary = needed_ballot_positions(doc, active_ballot_positions(doc).values())
+            ballot_summary = needed_ballot_positions(doc, doc.active_ballot().active_ad_positions().values())
 
         return render_to_response("idrfc/document_charter.html",
                                   dict(doc=doc,
@@ -171,7 +171,7 @@ def document_main(request, name, rev=None):
 
         ballot_summary = None
         if doc.get_state_slug() in ("iesgeval"):
-            ballot_summary = needed_ballot_positions(doc, active_ballot_positions(doc).values())
+            ballot_summary = needed_ballot_positions(doc, doc.active_ballot().active_ad_positions().values())
 
         return render_to_response("idrfc/document_conflict_review.html",
                                   dict(doc=doc,
@@ -292,36 +292,7 @@ def document_ballot_content(request, doc, ballot_id, editable=True):
 
     deferred = doc.active_defer_event()
 
-    # collect positions
-    active_ads = list(Person.objects.filter(role__name="ad", role__group__state="active").distinct())
-
-    positions = []
-    seen = {}
-    for e in BallotPositionDocEvent.objects.filter(doc=doc, type="changed_ballot_position", ballot=ballot).select_related('ad', 'pos').order_by("-time", '-id'):
-        if e.ad not in seen:
-            e.old_ad = e.ad not in active_ads
-            e.old_positions = []
-            positions.append(e)
-            seen[e.ad] = e
-        else:
-            latest = seen[e.ad]
-            if latest.old_positions:
-                prev = latest.old_positions[-1]
-            else:
-                prev = latest.pos.name
-
-            if e.pos.name != prev:
-                latest.old_positions.append(e.pos.name)
-
-    # add any missing ADs through fake No Record events
-    norecord = BallotPositionName.objects.get(slug="norecord")
-    for ad in active_ads:
-        if ad not in seen:
-            e = BallotPositionDocEvent(type="changed_ballot_position", doc=doc, ad=ad)
-            e.pos = norecord
-            e.old_ad = False
-            e.old_positions = []
-            positions.append(e)
+    positions = doc.active_ballot().all_positions() if doc.active_ballot() else []
 
     # put into position groups
     position_groups = []

ietf/idrfc/views_doc.py

@@ -146,6 +146,15 @@ def square_brackets(value):
     else:
         return "[   ]"
 
+@register.filter(name='bracketpos')
+def bracketpos(pos,posslug):
+    if pos.pos.slug==posslug:
+        return "[ X ]"
+    elif posslug in [x.slug for x in pos.old_positions]:
+        return "[ . ]"
+    else:
+        return "[   ]"
+
 @register.filter(name='fill')
 def fill(text, width):
     """Wraps each paragraph in text (a string) so every line