Merge in ^/branch/iola/event-saving-refactor-r10076, fixing a few problems

 - Legacy-Id: 10298

Author: OleLaursen

bin/test-crawl

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import os, sys, re, datetime, argparse, traceback, tempfile, json, subprocess
+import os, sys, re, datetime, argparse, traceback, json, subprocess
 import html5lib
 import random
 

ietf/bin/rfc-editor-index-updates

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import os, sys, re, json, datetime
+import os, sys, datetime
 import syslog
 import traceback
 
@@ -29,22 +29,28 @@ if options.skip_date:
     skip_date = datetime.datetime.strptime(options.skip_date, "%Y-%m-%d").date()
 
 from ietf.utils.pipe import pipe
-from ietf.sync.rfceditor import *
 from ietf.doc.utils import rebuild_reference_relations
+import ietf.sync.rfceditor
 
 syslog.syslog("Updating document metadata from RFC index from %s" % settings.RFC_EDITOR_QUEUE_URL)
 
-response = fetch_index_xml(settings.RFC_EDITOR_INDEX_URL)
-data = parse_index(response)
+response = ietf.sync.rfceditor.fetch_index_xml(settings.RFC_EDITOR_INDEX_URL)
+data = ietf.sync.rfceditor.parse_index(response)
 
-if len(data) < MIN_INDEX_RESULTS:
+if len(data) < ietf.sync.rfceditor.MIN_INDEX_RESULTS:
     syslog.syslog("Not enough results, only %s" % len(data))
     sys.exit(1)
 
-changed, new_rfcs  = update_docs_from_rfc_index(data, skip_older_than_date=skip_date)
+new_rfcs = []
+for changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_index(data, skip_older_than_date=skip_date):
+    if rfc_published:
+        new_rfcs.append(doc)
 
-for c in changed:
-    syslog.syslog(c)
+    for c in changes:
+        syslog.syslog("%s: %s" % (doc.name, c))
+        print "%s: %s" % (doc.name, c)
+
+sys.exit(0)
 
 # This can be called while processing a notifying POST from the RFC Editor
 # Spawn a child to sync the rfcs and calculate new reference relationships

ietf/doc/expire.py

@@ -6,7 +6,7 @@
 from pathlib import Path
 
 from ietf.utils.mail import send_mail
-from ietf.doc.models import Document, DocEvent, State, save_document_in_history, IESG_SUBSTATE_TAGS
+from ietf.doc.models import Document, DocEvent, State, IESG_SUBSTATE_TAGS
 from ietf.person.models import Person 
 from ietf.meeting.models import Meeting
 from ietf.doc.utils import add_state_change_event
@@ -129,8 +129,9 @@ def expire_draft(doc):
 
     system = Person.objects.get(name="(System)")
 
+    events = []
+
     # change the state
-    save_document_in_history(doc)
     if doc.latest_event(type='started_iesg_process'):
         new_state = State.objects.get(used=True, type="draft-iesg", slug="dead")
         prev_state = doc.get_state(new_state.type_id)
@@ -139,15 +140,13 @@ def expire_draft(doc):
             doc.set_state(new_state)
             doc.tags.remove(*prev_tags)
             e = add_state_change_event(doc, system, prev_state, new_state, prev_tags=prev_tags, new_tags=[])
+            if e:
+                events.append(e)
 
-        e = DocEvent(doc=doc, by=system)
-        e.type = "expired_document"
-        e.desc = "Document has expired"
-        e.save()
+    events.append(DocEvent.objects.create(doc=doc, by=system, type="expired_document", desc="Document has expired"))
 
     doc.set_state(State.objects.get(used=True, type="draft", slug="expired"))
-    doc.time = datetime.datetime.now()
-    doc.save()
+    doc.save_with_history(events)
 
 def clean_up_draft_files():
     """Move unidentified and old files out of the Internet Draft directory."""

ietf/doc/lastcall.py

@@ -5,7 +5,6 @@
 from django.db.models import Q
 
 from ietf.doc.models import Document, State, DocEvent, LastCallDocEvent, WriteupDocEvent
-from ietf.doc.models import save_document_in_history
 from ietf.doc.models import IESG_SUBSTATE_TAGS
 from ietf.person.models import Person
 from ietf.doc.utils import add_state_change_event
@@ -14,12 +13,15 @@
 
 def request_last_call(request, doc):
     if not doc.latest_event(type="changed_ballot_writeup_text"):
-        generate_ballot_writeup(request, doc)
+        e = generate_ballot_writeup(request, doc)
+        e.save()
     if not doc.latest_event(type="changed_ballot_approval_text"):
-        generate_approval_mail(request, doc)
+        e = generate_approval_mail(request, doc)
+        e.save()
     if not doc.latest_event(type="changed_last_call_text"):
-        generate_last_call_announcement(request, doc)
-    
+        e = generate_last_call_announcement(request, doc)
+        e.save()
+
     send_last_call_request(request, doc)
 
     e = DocEvent()
@@ -50,8 +52,6 @@ def expire_last_call(doc):
     else:
         raise ValueError("Unexpected document type to expire_last_call(): %s" % doc.type)
 
-    save_document_in_history(doc)
-
     prev_state = doc.get_state(new_state.type_id)
     doc.set_state(new_state)
 
@@ -60,8 +60,7 @@ def expire_last_call(doc):
 
     system = Person.objects.get(name="(System)")
     e = add_state_change_event(doc, system, prev_state, new_state, prev_tags=prev_tags, new_tags=[])
-                    
-    doc.time = (e and e.time) or datetime.datetime.now()
-    doc.save()
+    if e:
+        doc.save_with_history([e])
 
     email_last_call_expired(doc)

ietf/doc/mails.py

@@ -78,6 +78,21 @@ def email_iesg_processing_document(request, doc, changes):
 def html_to_text(html):
     return strip_tags(html.replace("&lt;", "<").replace("&gt;", ">").replace("&amp;", "&").replace("<br>", "\n"))
 
+def email_update_telechat(request, doc, text):
+    (to, cc) = gather_address_lists('doc_telechat_details_changed',doc=doc)
+
+    if not to:
+        return
+    
+    text = strip_tags(text)
+    send_mail(request, to, None,
+              "Telechat update notice: %s" % doc.file_tag(),
+              "doc/mail/update_telechat.txt",
+              dict(text=text,
+                   url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url()),
+              cc=cc)
+
+
 def generate_ballot_writeup(request, doc):
     e = doc.latest_event(type="iana_review")
     iana = e.desc if e else ""
@@ -88,8 +103,8 @@ def generate_ballot_writeup(request, doc):
     e.doc = doc
     e.desc = u"Ballot writeup was generated"
     e.text = unicode(render_to_string("doc/mail/ballot_writeup.txt", {'iana': iana}))
-    e.save()
-    
+
+    # caller is responsible for saving, if necessary
     return e
 
 def generate_last_call_announcement(request, doc):
@@ -130,8 +145,8 @@ def generate_last_call_announcement(request, doc):
     e.doc = doc
     e.desc = u"Last call announcement was generated"
     e.text = unicode(mail)
-    e.save()
 
+    # caller is responsible for saving, if necessary
     return e
 
 
@@ -149,8 +164,8 @@ def generate_approval_mail(request, doc):
     e.doc = doc
     e.desc = u"Ballot approval text was generated"
     e.text = unicode(mail)
-    e.save()
 
+    # caller is responsible for saving, if necessary
     return e
 
 def generate_approval_mail_approved(request, doc):

ietf/doc/migrations/0011_auto_20151027_1127.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+def fix_buggy_author_foreignkey(apps, schema_editor):
+    DocumentAuthor = apps.get_model("doc", "DocumentAuthor")
+    # apparently, we have a buggy key in the DB, fix it
+    DocumentAuthor.objects.filter(author="[<Email: d3e3e3@gmail.com>]").update(author="d3e3e3@gmail.com")
+
+def save_all_documents_in_history(apps, schema_editor):
+    State = apps.get_model("doc", "State")
+    Document = apps.get_model("doc", "Document")
+    DocHistory = apps.get_model("doc", "DocHistory")
+    RelatedDocument = apps.get_model("doc", "RelatedDocument")
+    RelatedDocHistory = apps.get_model("doc", "RelatedDocHistory")
+    DocumentAuthor = apps.get_model("doc", "DocumentAuthor")
+    DocHistoryAuthor = apps.get_model("doc", "DocHistoryAuthor")
+
+    def canonical_name(self):
+        name = self.name
+        state = State.objects.filter(document=self, type_id=self.type_id).first()
+        if self.type_id == "draft" and state.slug == "rfc":
+            a = self.docalias_set.filter(name__startswith="rfc")
+            if a:
+                name = a[0].name
+        elif self.type_id == "charter":
+            return charter_name_for_group(self.chartered_group)
+        return name
+
+    def charter_name_for_group(group):
+        if group.type_id == "rg":
+            top_org = "irtf"
+        else:
+            top_org = "ietf"
+
+        return "charter-%s-%s" % (top_org, group.acronym)
+
+    def save_document_in_history(doc):
+        """Save a snapshot of document and related objects in the database."""
+        def get_model_fields_as_dict(obj):
+            return dict((field.name, getattr(obj, field.name))
+                        for field in obj._meta.fields
+                        if field is not obj._meta.pk)
+
+        # copy fields
+        fields = get_model_fields_as_dict(doc)
+        fields["doc"] = doc
+        fields["name"] = canonical_name(doc)
+
+        dochist = DocHistory(**fields)
+        dochist.save()
+
+        # copy many to many
+        for field in doc._meta.many_to_many:
+            if field.rel.through and field.rel.through._meta.auto_created:
+                setattr(dochist, field.name, getattr(doc, field.name).all())
+
+        # copy remaining tricky many to many
+        def transfer_fields(obj, HistModel):
+            mfields = get_model_fields_as_dict(item)
+            # map doc -> dochist
+            for k, v in mfields.iteritems():
+                if v == doc:
+                    mfields[k] = dochist
+            HistModel.objects.create(**mfields)
+
+        for item in RelatedDocument.objects.filter(source=doc):
+            transfer_fields(item, RelatedDocHistory)
+
+        for item in DocumentAuthor.objects.filter(document=doc):
+            transfer_fields(item, DocHistoryAuthor)
+
+        return dochist
+
+    from django.conf import settings
+    settings.DEBUG = False # prevent out-of-memory problems
+
+    for d in Document.objects.iterator():
+        save_document_in_history(d)
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('doc', '0010_auto_20150930_0251'),
+    ]
+
+    operations = [
+        migrations.RunPython(fix_buggy_author_foreignkey),
+        migrations.RunPython(save_all_documents_in_history)
+    ]

ietf/doc/models.py

@@ -18,7 +18,6 @@
 from ietf.person.models import Email, Person
 from ietf.utils.admin import admin_link
 
-
 class StateType(models.Model):
     slug = models.CharField(primary_key=True, max_length=30) # draft, draft-iesg, charter, ...
     label = models.CharField(max_length=255, help_text="Label that should be used (e.g. in admin) for state drop-down for this type of state") # State, IESG state, WG state, ...
@@ -432,6 +431,27 @@ def display_name(self):
             name = name.upper()
         return name
 
+    def save_with_history(self, events):
+        """Save document and put a snapshot in the history models where they
+        can be retrieved later. You must pass in at least one event
+        with a description of what happened."""
+
+        assert events, "You must always add at least one event to describe the changes in the history log"
+        self.time = max(self.time, events[0].time)
+
+        self._has_an_event_so_saving_is_allowed = True
+        self.save()
+        del self._has_an_event_so_saving_is_allowed
+
+        from ietf.doc.utils import save_document_in_history
+        save_document_in_history(self)
+
+    def save(self, *args, **kwargs):
+        # if there's no primary key yet, we can allow the save to go
+        # through to break the cycle between the document and any
+        # events
+        assert kwargs.get("force_insert", False) or getattr(self, "_has_an_event_so_saving_is_allowed", None), "Use .save_with_history to save documents"
+        super(Document, self).save(*args, **kwargs)
 
     def telechat_date(self, e=None):
         if not e:
@@ -568,50 +588,6 @@ class Meta:
         verbose_name = "document history"
         verbose_name_plural = "document histories"
 
-def save_document_in_history(doc):
-    """This should be called before saving changes to a Document instance,
-    so that the DocHistory entries contain all previous states, while
-    the Group entry contain the current state.  XXX TODO: Call this
-    directly from Document.save(), and add event listeners for save()
-    on related objects so we can save as needed when they change, too.
-    """
-    def get_model_fields_as_dict(obj):
-        return dict((field.name, getattr(obj, field.name))
-                    for field in obj._meta.fields
-                    if field is not obj._meta.pk)
-
-    # copy fields
-    fields = get_model_fields_as_dict(doc)
-    fields["doc"] = doc
-    fields["name"] = doc.canonical_name()
-    
-    dochist = DocHistory(**fields)
-    dochist.save()
-
-    # copy many to many
-    for field in doc._meta.many_to_many:
-        if field.rel.through and field.rel.through._meta.auto_created:
-            setattr(dochist, field.name, getattr(doc, field.name).all())
-
-    # copy remaining tricky many to many
-    def transfer_fields(obj, HistModel):
-        mfields = get_model_fields_as_dict(item)
-        # map doc -> dochist
-        for k, v in mfields.iteritems():
-            if v == doc:
-                mfields[k] = dochist
-        HistModel.objects.create(**mfields)
-
-    for item in RelatedDocument.objects.filter(source=doc):
-        transfer_fields(item, RelatedDocHistory)
-
-    for item in DocumentAuthor.objects.filter(document=doc):
-        transfer_fields(item, DocHistoryAuthor)
-                
-    return dochist
-        
-
-
 class DocAlias(models.Model):
     """This is used for documents that may appear under multiple names,
     and in particular for RFCs, which for continuity still keep the
@@ -693,7 +669,8 @@ class DocReminder(models.Model):
 
     # RFC Editor
     ("rfc_editor_received_announcement", "Announcement was received by RFC Editor"),
-    ("requested_publication", "Publication at RFC Editor requested")
+    ("requested_publication", "Publication at RFC Editor requested"),
+    ("sync_from_rfc_editor", "Received updated information from RFC Editor"),
     ]
 
 class DocEvent(models.Model):
@@ -705,10 +682,11 @@ class DocEvent(models.Model):
     desc = models.TextField()
 
     def for_current_revision(self):
-        return self.time >= self.doc.latest_event(NewRevisionDocEvent,type='new_revision').time
+        e = self.doc.latest_event(NewRevisionDocEvent,type='new_revision')
+        return not e or (self.time, self.pk) >= (e.time, e.pk)
 
     def get_dochistory(self):
-        return DocHistory.objects.filter(time__lte=self.time,doc__name=self.doc.name).order_by('-time').first()
+        return DocHistory.objects.filter(time__lte=self.time,doc__name=self.doc.name).order_by('-time', '-pk').first()
 
     def __unicode__(self):
         return u"%s %s by %s at %s" % (self.doc.name, self.get_type_display().lower(), self.by.plain_name(), self.time)