Summary: Add new document saving API, Document.save_with_history(events).

The new API requires at least one event and will automatically save a
snapshot of the document and related state. Document.save() will now
throw an exception if called directly, as the new API is intended to
ensure that documents are saved with both an appropriate snapsnot and
relevant history log, both of which are easily defeated by just
calling .save() directly.

To simplify things, the snapshot is generated after the changes to a
document have been made (in anticipation of coming changes), instead
of before as was usual.

While revising the existing code to work with this API, a couple of
missing events was discovered:

- In draft expiry, a "Document has expired" event was only generated
  in case an IESG process had started on the document - now it's
  always generated, as the document changes its state in any case

- Synchronization updates like title and abstract amendmends from the
  RFC Editor were silently (except for RFC publication) applied and
  not accompanied by a descriptive event - they now are

- do_replace in the Secretariat tools now adds an event

- Proceedings post_process in the Secretariat tools now adds an event

- do_withdraw in the Secretariat tools now adds an event

A migration is needed for snapshotting all documents, takes a while to
run. It turns out that a single document had a bad foreign key so the
migration fixes that too.
 - Legacy-Id: 10101

Author: OleLaursen

ietf/bin/rfc-editor-index-updates

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import os, sys, re, json, datetime
+import os, sys, datetime
 import syslog
 import traceback
 
@@ -29,22 +29,28 @@ if options.skip_date:
     skip_date = datetime.datetime.strptime(options.skip_date, "%Y-%m-%d").date()
 
 from ietf.utils.pipe import pipe
-from ietf.sync.rfceditor import *
 from ietf.doc.utils import rebuild_reference_relations
+import ietf.sync.rfceditor
 
 syslog.syslog("Updating document metadata from RFC index from %s" % settings.RFC_EDITOR_QUEUE_URL)
 
-response = fetch_index_xml(settings.RFC_EDITOR_INDEX_URL)
-data = parse_index(response)
+response = ietf.sync.rfceditor.fetch_index_xml(settings.RFC_EDITOR_INDEX_URL)
+data = ietf.sync.rfceditor.parse_index(response)
 
-if len(data) < MIN_INDEX_RESULTS:
+if len(data) < ietf.sync.rfceditor.MIN_INDEX_RESULTS:
     syslog.syslog("Not enough results, only %s" % len(data))
     sys.exit(1)
 
-changed, new_rfcs  = update_docs_from_rfc_index(data, skip_older_than_date=skip_date)
+new_rfcs = []
+for changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_index(data, skip_older_than_date=skip_date):
+    if rfc_published:
+        new_rfcs.append(doc)
 
-for c in changed:
-    syslog.syslog(c)
+    for c in changes:
+        syslog.syslog("%s: %s" % (doc.name, c))
+        print "%s: %s" % (doc.name, c)
+
+sys.exit(0)
 
 # This can be called while processing a notifying POST from the RFC Editor
 # Spawn a child to sync the rfcs and calculate new reference relationships

ietf/doc/expire.py

@@ -6,7 +6,7 @@
 from pathlib import Path
 
 from ietf.utils.mail import send_mail
-from ietf.doc.models import Document, DocEvent, State, save_document_in_history, IESG_SUBSTATE_TAGS
+from ietf.doc.models import Document, DocEvent, State, IESG_SUBSTATE_TAGS
 from ietf.person.models import Person, Email
 from ietf.meeting.models import Meeting
 from ietf.doc.utils import add_state_change_event
@@ -131,8 +131,9 @@ def expire_draft(doc):
 
     system = Person.objects.get(name="(System)")
 
+    events = []
+
     # change the state
-    save_document_in_history(doc)
     if doc.latest_event(type='started_iesg_process'):
         new_state = State.objects.get(used=True, type="draft-iesg", slug="dead")
         prev_state = doc.get_state(new_state.type_id)
@@ -141,15 +142,13 @@ def expire_draft(doc):
             doc.set_state(new_state)
             doc.tags.remove(*prev_tags)
             e = add_state_change_event(doc, system, prev_state, new_state, prev_tags=prev_tags, new_tags=[])
+            if e:
+                events.append(e)
 
-        e = DocEvent(doc=doc, by=system)
-        e.type = "expired_document"
-        e.desc = "Document has expired"
-        e.save()
+    events.append(DocEvent.objects.create(doc=doc, by=system, type="expired_document", desc="Document has expired"))
 
     doc.set_state(State.objects.get(used=True, type="draft", slug="expired"))
-    doc.time = datetime.datetime.now()
-    doc.save()
+    doc.save_with_history(events)
 
 def clean_up_draft_files():
     """Move unidentified and old files out of the Internet Draft directory."""

ietf/doc/lastcall.py

@@ -5,7 +5,6 @@
 from django.db.models import Q
 
 from ietf.doc.models import Document, State, DocEvent, LastCallDocEvent, WriteupDocEvent
-from ietf.doc.models import save_document_in_history
 from ietf.doc.models import IESG_SUBSTATE_TAGS
 from ietf.person.models import Person
 from ietf.doc.utils import add_state_change_event
@@ -50,8 +49,6 @@ def expire_last_call(doc):
     else:
         raise ValueError("Unexpected document type to expire_last_call(): %s" % doc.type)
 
-    save_document_in_history(doc)
-
     prev_state = doc.get_state(new_state.type_id)
     doc.set_state(new_state)
 
@@ -60,8 +57,7 @@ def expire_last_call(doc):
 
     system = Person.objects.get(name="(System)")
     e = add_state_change_event(doc, system, prev_state, new_state, prev_tags=prev_tags, new_tags=[])
-                    
-    doc.time = (e and e.time) or datetime.datetime.now()
-    doc.save()
+    if e:
+        doc.save_with_history([e])
 
     email_last_call_expired(doc)

ietf/doc/mails.py

@@ -87,6 +87,20 @@ def email_ad(request, doc, ad, changed_by, text, subject=None):
                    doc=doc,
                    url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url()))
 
+def email_update_telechat(request, doc, text):
+    to = set(['iesg@ietf.org','iesg-secretary@ietf.org'])
+    to.update(set([x.strip() for x in doc.notify.replace(';', ',').split(',')]))
+
+    if not to:
+        return
+    
+    text = strip_tags(text)
+    send_mail(request, list(to), None,
+              "Telechat update notice: %s" % doc.file_tag(),
+              "doc/mail/update_telechat.txt",
+              dict(text=text,
+                   url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url()))
+
 
 def generate_ballot_writeup(request, doc):
     e = doc.latest_event(type="iana_review")

ietf/doc/migrations/0006_auto_20150924_0800.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+def fix_buggy_author_foreignkey(apps, schema_editor):
+    DocumentAuthor = apps.get_model("doc", "DocumentAuthor")
+    # apparently, we have a buggy key in the DB, fix it
+    DocumentAuthor.objects.filter(author="[<Email: d3e3e3@gmail.com>]").update(author="d3e3e3@gmail.com")
+
+def save_all_documents_in_history(apps, schema_editor):
+    State = apps.get_model("doc", "State")
+    Document = apps.get_model("doc", "Document")
+    DocHistory = apps.get_model("doc", "DocHistory")
+    RelatedDocument = apps.get_model("doc", "RelatedDocument")
+    RelatedDocHistory = apps.get_model("doc", "RelatedDocHistory")
+    DocumentAuthor = apps.get_model("doc", "DocumentAuthor")
+    DocHistoryAuthor = apps.get_model("doc", "DocHistoryAuthor")
+
+    def canonical_name(self):
+        name = self.name
+        state = State.objects.filter(document=self, type_id=self.type_id).first()
+        if self.type_id == "draft" and state.slug == "rfc":
+            a = self.docalias_set.filter(name__startswith="rfc")
+            if a:
+                name = a[0].name
+        elif self.type_id == "charter":
+            return charter_name_for_group(self.chartered_group)
+        return name
+
+    def charter_name_for_group(group):
+        if group.type_id == "rg":
+            top_org = "irtf"
+        else:
+            top_org = "ietf"
+
+        return "charter-%s-%s" % (top_org, group.acronym)
+
+    def save_document_in_history(doc):
+        """Save a snapshot of document and related objects in the database."""
+        def get_model_fields_as_dict(obj):
+            return dict((field.name, getattr(obj, field.name))
+                        for field in obj._meta.fields
+                        if field is not obj._meta.pk)
+
+        # copy fields
+        fields = get_model_fields_as_dict(doc)
+        fields["doc"] = doc
+        fields["name"] = canonical_name(doc)
+
+        dochist = DocHistory(**fields)
+        dochist.save()
+
+        # copy many to many
+        for field in doc._meta.many_to_many:
+            if field.rel.through and field.rel.through._meta.auto_created:
+                setattr(dochist, field.name, getattr(doc, field.name).all())
+
+        # copy remaining tricky many to many
+        def transfer_fields(obj, HistModel):
+            mfields = get_model_fields_as_dict(item)
+            # map doc -> dochist
+            for k, v in mfields.iteritems():
+                if v == doc:
+                    mfields[k] = dochist
+            HistModel.objects.create(**mfields)
+
+        for item in RelatedDocument.objects.filter(source=doc):
+            transfer_fields(item, RelatedDocHistory)
+
+        for item in DocumentAuthor.objects.filter(document=doc):
+            transfer_fields(item, DocHistoryAuthor)
+
+        return dochist
+
+    from django.conf import settings
+    settings.DEBUG = False # prevent out-of-memory problems
+
+    for d in Document.objects.iterator():
+        save_document_in_history(d)
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('doc', '0005_auto_20150721_0230'),
+    ]
+
+    operations = [
+        migrations.RunPython(fix_buggy_author_foreignkey),
+        migrations.RunPython(save_all_documents_in_history)
+    ]

ietf/doc/models.py

@@ -18,7 +18,6 @@
 from ietf.person.models import Email, Person
 from ietf.utils.admin import admin_link
 
-
 class StateType(models.Model):
     slug = models.CharField(primary_key=True, max_length=30) # draft, draft-iesg, charter, ...
     label = models.CharField(max_length=255, help_text="Label that should be used (e.g. in admin) for state drop-down for this type of state") # State, IESG state, WG state, ...
@@ -433,6 +432,27 @@ def display_name(self):
             name = name.upper()
         return name
 
+    def save_with_history(self, events):
+        """Save document and put a snapshot in the history models where they
+        can be retrieved later. You must pass in at least one event
+        with a description of what happened."""
+
+        assert events, "You must always add at least one event to describe the changes in the history log"
+        self.time = max(self.time, events[0].time)
+
+        self._has_an_event_so_saving_is_allowed = True
+        self.save()
+        del self._has_an_event_so_saving_is_allowed
+
+        from ietf.doc.utils import save_document_in_history
+        save_document_in_history(self)
+
+    def save(self, *args, **kwargs):
+        # if there's no primary key yet, we can allow the save to go
+        # through to break the cycle between the document and any
+        # events
+        assert kwargs.get("force_insert", False) or getattr(self, "_has_an_event_so_saving_is_allowed", None), "Use .save_with_history to save documents"
+        super(Document, self).save(*args, **kwargs)
 
     def telechat_date(self, e=None):
         if not e:
@@ -570,50 +590,6 @@ class Meta:
         verbose_name = "document history"
         verbose_name_plural = "document histories"
 
-def save_document_in_history(doc):
-    """This should be called before saving changes to a Document instance,
-    so that the DocHistory entries contain all previous states, while
-    the Group entry contain the current state.  XXX TODO: Call this
-    directly from Document.save(), and add event listeners for save()
-    on related objects so we can save as needed when they change, too.
-    """
-    def get_model_fields_as_dict(obj):
-        return dict((field.name, getattr(obj, field.name))
-                    for field in obj._meta.fields
-                    if field is not obj._meta.pk)
-
-    # copy fields
-    fields = get_model_fields_as_dict(doc)
-    fields["doc"] = doc
-    fields["name"] = doc.canonical_name()
-    
-    dochist = DocHistory(**fields)
-    dochist.save()
-
-    # copy many to many
-    for field in doc._meta.many_to_many:
-        if field.rel.through and field.rel.through._meta.auto_created:
-            setattr(dochist, field.name, getattr(doc, field.name).all())
-
-    # copy remaining tricky many to many
-    def transfer_fields(obj, HistModel):
-        mfields = get_model_fields_as_dict(item)
-        # map doc -> dochist
-        for k, v in mfields.iteritems():
-            if v == doc:
-                mfields[k] = dochist
-        HistModel.objects.create(**mfields)
-
-    for item in RelatedDocument.objects.filter(source=doc):
-        transfer_fields(item, RelatedDocHistory)
-
-    for item in DocumentAuthor.objects.filter(document=doc):
-        transfer_fields(item, DocHistoryAuthor)
-                
-    return dochist
-        
-
-
 class DocAlias(models.Model):
     """This is used for documents that may appear under multiple names,
     and in particular for RFCs, which for continuity still keep the
@@ -695,7 +671,8 @@ class DocReminder(models.Model):
 
     # RFC Editor
     ("rfc_editor_received_announcement", "Announcement was received by RFC Editor"),
-    ("requested_publication", "Publication at RFC Editor requested")
+    ("requested_publication", "Publication at RFC Editor requested"),
+    ("sync_from_rfc_editor", "Received updated information from RFC Editor"),
     ]
 
 class DocEvent(models.Model):

ietf/doc/tests.py

@@ -17,8 +17,7 @@
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import ( Document, DocAlias, DocRelationshipName, RelatedDocument, State,
-    DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent,
-    save_document_in_history )
+    DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent )
 from ietf.group.models import Group
 from ietf.meeting.models import Meeting, Session, SessionPresentation
 from ietf.name.models import SessionStatusName
@@ -432,9 +431,8 @@ def test_document_draft(self):
         # draft published as RFC
         draft.set_state(State.objects.get(type="draft", slug="rfc"))
         draft.std_level_id = "bcp"
-        draft.save()
+        draft.save_with_history([DocEvent.objects.create(doc=draft, type="published_rfc", by=Person.objects.get(name="(System)"))])
 
-        DocEvent.objects.create(doc=draft, type="published_rfc", by=Person.objects.get(name="(System)"))
 
         rfc_alias = DocAlias.objects.create(name="rfc123456", document=draft)
         bcp_alias = DocAlias.objects.create(name="bcp123456", document=draft)
@@ -480,9 +478,10 @@ def test_document_primary_and_history_views(self):
                        ]:
             doc = Document.objects.get(name=docname)
             # give it some history
-            save_document_in_history(doc)
+            doc.save_with_history([DocEvent(doc=doc)])
+
             doc.rev="01"
-            doc.save()
+            doc.save_with_history([DocEvent(doc=doc)])
 
             r = self.client.get(urlreverse("doc_view", kwargs=dict(name=doc.name)))
             self.assertEqual(r.status_code, 200)
@@ -539,7 +538,8 @@ def test_document_ballot(self):
         doc = make_test_data()
         ballot = doc.active_ballot()
 
-        save_document_in_history(doc)
+        # make sure we have some history
+        doc.save_with_history([DocEvent.objects.create(doc=doc, type="added_comment", by=Person.objects.get(user__username="secretary"), desc="Test")])
 
         pos = BallotPositionDocEvent.objects.create(
             doc=doc,
@@ -567,9 +567,8 @@ def test_document_ballot(self):
         # Now simulate a new revision and make sure positions on older revisions are marked as such
         oldrev = doc.rev
         e = NewRevisionDocEvent.objects.create(doc=doc,rev='%02d'%(int(doc.rev)+1),type='new_revision',by=Person.objects.get(name="(System)"))
-        save_document_in_history(doc)
         doc.rev = e.rev
-        doc.save()
+        doc.save_with_history([e])
         r = self.client.get(urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name)))
         self.assertEqual(r.status_code, 200)
         self.assertTrue( '(%s for -%s)' % (pos.comment_time.strftime('%Y-%m-%d'), oldrev) in r.content)