Merged in [15822] and [15809] from rjsparks@nostrum.com:

    Added an explicit ID-Exists state for the IESG state machine.
    Reworked code so that the IESG state machine always has a state.
    Added the ability to release a document from a working group,
    research group, or the independent stream. Releasing a document
    removes all stream state, and sets the document to have no stream.
 - Legacy-Id: 15823
Note: SVN reference [15809] has been migrated to Git commit 6118975

Note: SVN reference [15822] has been migrated to Git commit f6886f6

Author: levkowetz

ietf/doc/expire.py

@@ -5,6 +5,7 @@
 import datetime, os, shutil, glob, re
 from pathlib import Path
 
+from ietf.utils import log
 from ietf.utils.mail import send_mail
 from ietf.doc.models import Document, DocEvent, State, IESG_SUBSTATE_TAGS
 from ietf.person.models import Person 
@@ -17,8 +18,11 @@ def expirable_draft(draft):
     """Return whether draft is in an expirable state or not. This is
     the single draft version of the logic in expirable_drafts. These
     two functions need to be kept in sync."""
+    if draft.type_id != 'draft':
+        return False
+    log.assertion('draft.get_state_slug("draft-iesg")')
     return (draft.expires and draft.get_state_slug() == "active"
-            and draft.get_state_slug("draft-iesg") in (None, "watching", "dead")
+            and draft.get_state_slug("draft-iesg") in ("idexists", "watching", "dead")
             and draft.get_state_slug("draft-stream-%s" % draft.stream_id) not in ("rfc-edit", "pub")
             and not draft.tags.filter(slug="rfc-rev"))
 
@@ -29,8 +33,8 @@ def expirable_drafts():
     d = Document.objects.filter(states__type="draft", states__slug="active").exclude(expires=None)
 
     nonexpirable_states = []
-    # all IESG states except AD Watching and Dead block expiry
-    nonexpirable_states += list(State.objects.filter(used=True, type="draft-iesg").exclude(slug__in=("watching", "dead")))
+    # all IESG states except I-D Exists, AD Watching, and Dead block expiry
+    nonexpirable_states += list(State.objects.filter(used=True, type="draft-iesg").exclude(slug__in=("idexists","watching", "dead")))
     # sent to RFC Editor and RFC Published block expiry (the latter
     # shouldn't be possible for an active draft, though)
     nonexpirable_states += list(State.objects.filter(used=True, type__in=("draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"), slug__in=("rfc-edit", "pub")))
@@ -75,7 +79,8 @@ def send_expire_warning_for_draft(doc):
     (to,cc) = gather_address_lists('doc_expires_soon',doc=doc)
 
     s = doc.get_state("draft-iesg")
-    state = s.name if s else "I-D Exists"
+    log.assertion('s')
+    state = s.name if s else "I-D Exists" # TODO remove the if clause after some runtime shows no assertions
 
     frm = None
     request = None
@@ -94,7 +99,8 @@ def send_expire_notice_for_draft(doc):
         return
 
     s = doc.get_state("draft-iesg")
-    state = s.name if s else "I-D Exists"
+    log.assertion('s')
+    state = s.name if s else "I-D Exists" # TODO remove the if clause after some rintime shows no assertions
 
     request = None
     (to,cc) = gather_address_lists('doc_expired',doc=doc)

ietf/doc/factories.py

@@ -46,6 +46,9 @@ def states(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument
         if create and extracted:
             for (state_type_id,state_slug) in extracted:
                 obj.set_state(State.objects.get(type_id=state_type_id,slug=state_slug))
+            if obj.type_id == 'draft':
+                if not obj.states.filter(type_id='draft-iesg').exists():
+                    obj.set_state(State.objects.get(type_id='draft-iesg', slug='idexists'))
 
     @factory.post_generation
     def authors(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument
@@ -90,8 +93,11 @@ def states(obj, create, extracted, **kwargs):
         if extracted:
             for (state_type_id,state_slug) in extracted:
                 obj.set_state(State.objects.get(type_id=state_type_id,slug=state_slug))
+            if not obj.get_state('draft-iesg'):
+                obj.set_state(State.objects.get(type_id='draft-iesg',slug='idexists'))
         else:
             obj.set_state(State.objects.get(type_id='draft',slug='active'))
+            obj.set_state(State.objects.get(type_id='draft-iesg',slug='idexists'))
 
 class IndividualRfcFactory(IndividualDraftFactory):
 
@@ -120,9 +126,12 @@ def states(obj, create, extracted, **kwargs):
         if extracted:
             for (state_type_id,state_slug) in extracted:
                 obj.set_state(State.objects.get(type_id=state_type_id,slug=state_slug))
+            if not obj.get_state('draft-iesg'):
+                obj.set_state(State.objects.get(type_id='draft-iesg',slug='idexists'))
         else:
             obj.set_state(State.objects.get(type_id='draft',slug='active'))
             obj.set_state(State.objects.get(type_id='draft-stream-ietf',slug='wg-doc'))
+            obj.set_state(State.objects.get(type_id='draft-iesg',slug='idexists'))
 
 class WgRfcFactory(WgDraftFactory):
 
@@ -137,8 +146,33 @@ def states(obj, create, extracted, **kwargs):
         if extracted:
             for (state_type_id,state_slug) in extracted:
                 obj.set_state(State.objects.get(type_id=state_type_id,slug=state_slug))
+            if not obj.get_state('draft-iesg'):
+                obj.set_state(State.objects.get(type_id='draft-iesg', slug='pub'))
         else:
             obj.set_state(State.objects.get(type_id='draft',slug='rfc'))
+            obj.set_state(State.objects.get(type_id='draft-iesg', slug='pub'))
+
+
+class RgDraftFactory(BaseDocumentFactory):
+
+    type_id = 'draft'
+    group = factory.SubFactory('ietf.group.factories.GroupFactory',type_id='rg')
+    stream_id = 'irtf'
+
+    @factory.post_generation
+    def states(obj, create, extracted, **kwargs):
+        if not create:
+            return
+        if extracted:
+            for (state_type_id,state_slug) in extracted:
+                obj.set_state(State.objects.get(type_id=state_type_id,slug=state_slug))
+            if not obj.get_state('draft-iesg'):
+                obj.set_state(State.objects.get(type_id='draft-iesg',slug='idexists'))
+        else:
+            obj.set_state(State.objects.get(type_id='draft',slug='active'))
+            obj.set_state(State.objects.get(type_id='draft-stream-irtf',slug='active'))
+            obj.set_state(State.objects.get(type_id='draft-iesg',slug='idexists'))
+
 
 class CharterFactory(BaseDocumentFactory):
 

ietf/doc/migrations/0007_idexists.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.16 on 2018-11-04 10:56
+from __future__ import unicode_literals
+
+from tqdm import tqdm
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+    State = apps.get_model('doc','State')
+    Document = apps.get_model('doc','Document')
+    DocHistory = apps.get_model('doc','DocHistory')
+
+    idexists = State.objects.create(
+        type_id = 'draft-iesg',
+        slug = 'idexists',
+        name = 'I-D Exists',
+        used = True,
+        desc = 'The IESG has not started processing this draft, or has stopped processing it without publicastion.',
+        order = 0,
+    )
+    idexists.next_states.set(State.objects.filter(type='draft-iesg',slug__in=['pub-req','watching']))
+
+    #for doc in tqdm(Document.objects.filter(type='draft'):
+    #    if not doc.states.filter(type='draft-iesg').exists():
+    #        doc.states.add(idexists)
+    #    for dh in doc.history_set.all():
+    #        if not dh.states.filter(type='draft-iesg').exists():
+    #            dh.states.add(idexists)            
+
+    for doc in tqdm(Document.objects.filter(type_id='draft').exclude(states__type_id='draft-iesg')):
+        doc.states.add(idexists)
+    for history in tqdm(DocHistory.objects.filter(type_id='draft').exclude(states__type_id='draft-iesg')):
+        history.states.add(idexists)
+
+
+def reverse(apps, schema_editor):
+    State = apps.get_model('doc','State')
+    State.objects.filter(slug='idexists').delete()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('doc', '0006_ballotpositiondocevent_send_email'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse)
+    ]

ietf/doc/models.py

@@ -267,6 +267,7 @@ def set_state(self, state):
 
     def unset_state(self, state_type):
         """Unset state of type so no state of that type is any longer set."""
+        log.assertion('state_type != "draft-iesg"')
         self.states.remove(*self.states.filter(type=state_type))
         self.state_cache = None # invalidate cache
         self._cached_state_slug = {}
@@ -325,6 +326,7 @@ def friendly_state(self):
                 else:
                     return "Replaced"
             elif state.slug == "active":
+                log.assertion('iesg_state')
                 if iesg_state:
                     if iesg_state.slug == "dead":
                         # Many drafts in the draft-iesg "Dead" state are not dead

ietf/doc/templatetags/ballot_icon.py

@@ -42,6 +42,7 @@
 from ietf.ietfauth.utils import user_is_person, has_role
 from ietf.doc.models import BallotPositionDocEvent, IESG_BALLOT_ACTIVE_STATES
 from ietf.name.models import BallotPositionName
+from ietf.utils import log
 
 
 register = template.Library()
@@ -157,10 +158,11 @@ def state_age_colored(doc):
             # Don't show anything for expired/withdrawn/replaced drafts
             return ""
         iesg_state = doc.get_state_slug('draft-iesg')
+        log.assertion('iesg_state')
         if not iesg_state:
             return ""
 
-        if iesg_state in ["dead", "watching", "pub"]:
+        if iesg_state in ["dead", "watching", "pub", "idexists"]:
             return ""
         try:
             state_date = doc.docevent_set.filter(

ietf/doc/tests_draft.py

@@ -4,20 +4,21 @@
 import datetime
 import StringIO
 from pyquery import PyQuery
+from collections import Counter
 
 from django.urls import reverse as urlreverse
 from django.conf import settings
 
 import debug                            # pyflakes:ignore
 
-from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, DocEventFactory
+from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, RgDraftFactory, DocEventFactory
 from ietf.doc.models import ( Document, DocReminder, DocEvent,
     ConsensusDocEvent, LastCallDocEvent, RelatedDocument, State, TelechatDocEvent, 
     WriteupDocEvent, DocRelationshipName)
 from ietf.doc.utils import get_tags_for_stream_id, create_ballot_if_not_open
 from ietf.name.models import StreamName, DocTagName
 from ietf.group.factories import GroupFactory, RoleFactory
-from ietf.group.models import Group
+from ietf.group.models import Group, Role
 from ietf.person.factories import PersonFactory
 from ietf.person.models import Person, Email
 from ietf.meeting.models import Meeting, MeetingTypeName
@@ -419,7 +420,7 @@ def test_start_iesg_process_on_draft(self):
         self.assertTrue('draft-ietf-mars-test2@' in outbox[-1]['To']) 
 
         # Redo, starting in publication requested to make sure WG state is also set
-        draft.unset_state('draft-iesg')
+        draft.set_state(State.objects.get(type_id='draft-iesg', slug='idexists'))
         draft.set_state(State.objects.get(type='draft-stream-ietf',slug='writeupw'))
         draft.stream = StreamName.objects.get(slug='ietf')
         draft.save_with_history([DocEvent.objects.create(doc=draft, rev=draft.rev, type="changed_stream", by=Person.objects.get(user__username="secretary"), desc="Test")])
@@ -589,7 +590,7 @@ def test_warn_expirable_drafts(self):
         self.assertEqual(len(list(get_soon_to_expire_drafts(14))), 0)
 
         # hack into expirable state
-        draft.unset_state("draft-iesg")
+        draft.set_state(State.objects.get(type_id='draft-iesg',slug='idexists'))
         draft.expires = datetime.datetime.now() + datetime.timedelta(days=10)
         draft.save_with_history([DocEvent.objects.create(doc=draft, rev=draft.rev, type="changed_document", by=Person.objects.get(user__username="secretary"), desc="Test")])
 
@@ -616,7 +617,7 @@ def test_expire_drafts(self):
         self.assertEqual(len(list(get_expired_drafts())), 0)
 
         # hack into expirable state
-        draft.unset_state("draft-iesg")
+        draft.set_state(State.objects.get(type_id='draft-iesg',slug='idexists'))
         draft.expires = datetime.datetime.now()
         draft.save_with_history([DocEvent.objects.create(doc=draft, rev=draft.rev, type="changed_document", by=Person.objects.get(user__username="secretary"), desc="Test")])
 
@@ -1110,21 +1111,21 @@ def test_cancel_submission(self):
         url = urlreverse('ietf.doc.views_draft.to_iesg', kwargs=dict(name=self.docname))
         self.client.login(username="marschairman", password="marschairman+password")
 
-	r = self.client.post(url, dict(cancel="1"))
+        r = self.client.post(url, dict(cancel="1"))
         self.assertEqual(r.status_code, 302)
 
         doc = Document.objects.get(name=self.docname)
-        self.assertTrue(doc.get_state('draft-iesg')==None)
+        self.assertEqual(doc.get_state_slug('draft-iesg'),'idexists')
 
     def test_confirm_submission(self):
         url = urlreverse('ietf.doc.views_draft.to_iesg', kwargs=dict(name=self.docname))
         self.client.login(username="marschairman", password="marschairman+password")
 
         doc = Document.objects.get(name=self.docname)
-        docevent_count_pre = doc.docevent_set.count()
+        docevents_pre = set(doc.docevent_set.all())
         mailbox_before = len(outbox)
 
-	r = self.client.post(url, dict(confirm="1"))
+        r = self.client.post(url, dict(confirm="1"))
         self.assertEqual(r.status_code, 302)
 
         doc = Document.objects.get(name=self.docname)
@@ -1136,7 +1137,12 @@ def test_confirm_submission(self):
         # checks whether the setup document had an ad or not.
         self.assertTrue(doc.ad!=None)
 
-        self.assertTrue(doc.docevent_set.count() != docevent_count_pre)
+        new_docevents = set(doc.docevent_set.all()) - docevents_pre
+        self.assertEqual(len(new_docevents),3)
+        new_docevent_type_count = Counter([e.type for e in new_docevents])
+        self.assertEqual(new_docevent_type_count['changed_state'],2)
+        self.assertEqual(new_docevent_type_count['started_iesg_process'],1)
+
         self.assertEqual(len(outbox), mailbox_before + 1)
         self.assertTrue("Publication has been requested" in outbox[-1]['Subject'])
         self.assertTrue("aread@" in outbox[-1]['To'])
@@ -1180,6 +1186,88 @@ def test_request_publication(self):
 
         self.assertTrue("Document Action" in draft.message_set.order_by("-time")[0].subject)
 
+class ReleaseDraftTests(TestCase):
+    def test_release_wg_draft(self):
+        chair_role = RoleFactory(group__type_id='wg',name_id='chair') 
+        draft = WgDraftFactory(group = chair_role.group)
+        draft.tags.set(DocTagName.objects.filter(slug__in=('sh-f-up','w-merge'))) 
+        other_chair_role = RoleFactory(group__type_id='wg',name_id='chair')
+
+        url = urlreverse('ietf.doc.views_draft.release_draft', kwargs=dict(name=draft.name))
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 302) # redirect to login
+
+        self.client.login(username=other_chair_role.person.user.username,password=other_chair_role.person.user.username+"+password")
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 403)
+
+        self.client.logout()
+        self.client.login(username=chair_role.person.user.username, password=chair_role.person.user.username+"+password")
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+
+        events_before = list(draft.docevent_set.all())
+        empty_outbox()
+        r = self.client.post(url,{"comment": "Here are some comments"})
+        self.assertEqual(r.status_code, 302)
+        draft = Document.objects.get(pk=draft.pk)
+        self.assertEqual(draft.stream, None)
+        self.assertEqual(draft.group.type_id, "individ")
+        self.assertFalse(draft.get_state('draft-stream-ietf'))
+        self.assertEqual(len(outbox),3)
+        subjects = [msg["Subject"] for msg in outbox]
+        cat_subjects = "".join(subjects)
+        self.assertIn("Tags changed", cat_subjects)
+        self.assertIn("State Update", cat_subjects)
+        self.assertIn("Stream Change", cat_subjects)
+        descs = sorted([event.desc for event in set(list(draft.docevent_set.all())) - set(events_before)])
+        self.assertEqual("Changed stream to <b>None</b> from IETF",descs[0])
+        self.assertEqual("Here are some comments",descs[1])
+        self.assertEqual("State changed to <b>None</b> from WG Document",descs[2])
+        self.assertEqual("Tags Awaiting Merge with Other Document, Document Shepherd Followup cleared.",descs[3])
+
+    def test_release_rg_draft(self):
+        chair_role = RoleFactory(group__type_id='rg',name_id='chair')
+        draft = RgDraftFactory(group = chair_role.group)
+        url = urlreverse('ietf.doc.views_draft.release_draft', kwargs=dict(name=draft.name))
+        self.client.login(username=chair_role.person.user.username, password=chair_role.person.user.username+"+password")
+        r = self.client.post(url,{"comment": "Here are some comments"})
+        self.assertEqual(r.status_code, 302)
+        draft = Document.objects.get(pk=draft.pk)
+        self.assertEqual(draft.stream, None)
+        self.assertEqual(draft.group.type_id, "individ")
+        self.assertFalse(draft.get_state('draft-stream-irtf'))
+
+    def test_release_ise_draft(self):
+        ise = Role.objects.get(name_id='chair', group__acronym='ise')
+        draft = IndividualDraftFactory(stream_id='ise')
+        draft.set_state(State.objects.get(type_id='draft-stream-ise',slug='ise-rev'))
+        draft.tags.set(DocTagName.objects.filter(slug='w-dep'))
+        url = urlreverse('ietf.doc.views_draft.release_draft', kwargs=dict(name=draft.name))
+
+        self.client.login(username=ise.person.user.username, password=ise.person.user.username+'+password')
+
+        events_before = list(draft.docevent_set.all())
+        empty_outbox()
+        r = self.client.post(url,{"comment": "Here are some comments"})
+        self.assertEqual(r.status_code, 302)
+        draft = Document.objects.get(pk=draft.pk)
+        self.assertEqual(draft.stream, None)
+        self.assertEqual(draft.group.type_id, "individ")
+        self.assertFalse(draft.get_state('draft-stream-ise'))
+        self.assertEqual(len(outbox),3)
+        subjects = [msg["Subject"] for msg in outbox]
+        cat_subjects = "".join(subjects)
+        self.assertIn("Tags changed", cat_subjects)
+        self.assertIn("State Update", cat_subjects)
+        self.assertIn("Stream Change", cat_subjects)
+        descs = sorted([event.desc for event in set(list(draft.docevent_set.all())) - set(events_before)])
+        self.assertEqual("Changed stream to <b>None</b> from ISE",descs[0])
+        self.assertEqual("Here are some comments",descs[1])
+        self.assertEqual("State changed to <b>None</b> from In ISE Review",descs[2])
+        self.assertEqual("Tag Waiting for Dependency on Other Document cleared.",descs[3])
+
 class AdoptDraftTests(TestCase):
     def test_adopt_document(self):
         RoleFactory(group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',name_id='chair')

ietf/doc/urls.py

@@ -107,6 +107,7 @@
     url(r'^%(name)s/edit/shepherdwriteup/$' % settings.URL_REGEXPS, views_draft.edit_shepherd_writeup),
     url(r'^%(name)s/edit/requestpublication/$' % settings.URL_REGEXPS, views_draft.request_publication),
     url(r'^%(name)s/edit/adopt/$' % settings.URL_REGEXPS, views_draft.adopt_draft),
+    url(r'^%(name)s/edit/release/$' % settings.URL_REGEXPS, views_draft.release_draft),
     url(r'^%(name)s/edit/state/(?P<state_type>draft-stream-[a-z]+)/$' % settings.URL_REGEXPS, views_draft.change_stream_state),
 
     url(r'^%(name)s/edit/clearballot/(?P<ballot_type_slug>[\w-]+)/$' % settings.URL_REGEXPS, views_ballot.clear_ballot),