Merged in ^/branch/iola/event-saving-refactor-r10291 (via ^/personal/henrik/6.30.1-eventsave), which refactors document saving to always use doc.save_with_history(events), and requires accompanying events. This branch also provides refactoring of recurring regexes in url patterns into a dictionary. As part of the merge, also refactored new code which didn't use the save_with_history() method.

 - Legacy-Id: 11841

Author: levkowetz

bin/test-crawl

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import os, sys, re, datetime, argparse, traceback, tempfile, json, subprocess
+import os, sys, re, datetime, argparse, traceback, json, subprocess
 import html5lib
 import random
 

ietf/bin/rfc-editor-index-updates

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import os, sys, re, json, datetime
+import os, sys, datetime
 import syslog
 import traceback
 
@@ -33,22 +33,28 @@ if options.skip_date:
     skip_date = datetime.datetime.strptime(options.skip_date, "%Y-%m-%d").date()
 
 from ietf.utils.pipe import pipe
-from ietf.sync.rfceditor import *
 from ietf.doc.utils import rebuild_reference_relations
+import ietf.sync.rfceditor
 
 syslog.syslog("Updating document metadata from RFC index from %s" % settings.RFC_EDITOR_QUEUE_URL)
 
-response = fetch_index_xml(settings.RFC_EDITOR_INDEX_URL)
-data = parse_index(response)
+response = ietf.sync.rfceditor.fetch_index_xml(settings.RFC_EDITOR_INDEX_URL)
+data = ietf.sync.rfceditor.parse_index(response)
 
-if len(data) < MIN_INDEX_RESULTS:
+if len(data) < ietf.sync.rfceditor.MIN_INDEX_RESULTS:
     syslog.syslog("Not enough results, only %s" % len(data))
     sys.exit(1)
 
-changed, new_rfcs  = update_docs_from_rfc_index(data, skip_older_than_date=skip_date)
+new_rfcs = []
+for changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_index(data, skip_older_than_date=skip_date):
+    if rfc_published:
+        new_rfcs.append(doc)
 
-for c in changed:
-    syslog.syslog(c)
+    for c in changes:
+        syslog.syslog("%s: %s" % (doc.name, c))
+        print "%s: %s" % (doc.name, c)
+
+sys.exit(0)
 
 # This can be called while processing a notifying POST from the RFC Editor
 # Spawn a child to sync the rfcs and calculate new reference relationships

ietf/doc/expire.py

@@ -6,7 +6,7 @@
 from pathlib import Path
 
 from ietf.utils.mail import send_mail
-from ietf.doc.models import Document, DocEvent, State, save_document_in_history, IESG_SUBSTATE_TAGS
+from ietf.doc.models import Document, DocEvent, State, IESG_SUBSTATE_TAGS
 from ietf.person.models import Person 
 from ietf.meeting.models import Meeting
 from ietf.doc.utils import add_state_change_event
@@ -129,8 +129,9 @@ def expire_draft(doc):
 
     system = Person.objects.get(name="(System)")
 
+    events = []
+
     # change the state
-    save_document_in_history(doc)
     if doc.latest_event(type='started_iesg_process'):
         new_state = State.objects.get(used=True, type="draft-iesg", slug="dead")
         prev_state = doc.get_state(new_state.type_id)
@@ -139,15 +140,13 @@ def expire_draft(doc):
             doc.set_state(new_state)
             doc.tags.remove(*prev_tags)
             e = add_state_change_event(doc, system, prev_state, new_state, prev_tags=prev_tags, new_tags=[])
+            if e:
+                events.append(e)
 
-        e = DocEvent(doc=doc, by=system)
-        e.type = "expired_document"
-        e.desc = "Document has expired"
-        e.save()
+    events.append(DocEvent.objects.create(doc=doc, by=system, type="expired_document", desc="Document has expired"))
 
     doc.set_state(State.objects.get(used=True, type="draft", slug="expired"))
-    doc.time = datetime.datetime.now()
-    doc.save()
+    doc.save_with_history(events)
 
 def clean_up_draft_files():
     """Move unidentified and old files out of the Internet Draft directory."""

ietf/doc/factories.py

@@ -42,6 +42,15 @@ def states(self, create, extracted, **kwargs):
             for (state_type_id,state_slug) in extracted:
                 self.set_state(State.objects.get(type_id=state_type_id,slug=state_slug))
 
+    @classmethod
+    def _after_postgeneration(cls, obj, create, results=None):
+        """Save again the instance if creating and at least one hook ran."""
+        if create and results:
+            # Some post-generation hooks ran, and may have modified us.
+            obj._has_an_event_so_saving_is_allowed = True
+            obj.save()
+
+
 class DocAliasFactory(factory.DjangoModelFactory):
     class Meta:
         model = DocAlias

ietf/doc/lastcall.py

@@ -5,7 +5,6 @@
 from django.db.models import Q
 
 from ietf.doc.models import Document, State, DocEvent, LastCallDocEvent, WriteupDocEvent
-from ietf.doc.models import save_document_in_history
 from ietf.doc.models import IESG_SUBSTATE_TAGS
 from ietf.person.models import Person
 from ietf.doc.utils import add_state_change_event
@@ -14,12 +13,15 @@
 
 def request_last_call(request, doc):
     if not doc.latest_event(type="changed_ballot_writeup_text"):
-        generate_ballot_writeup(request, doc)
+        e = generate_ballot_writeup(request, doc)
+        e.save()
     if not doc.latest_event(type="changed_ballot_approval_text"):
-        generate_approval_mail(request, doc)
+        e = generate_approval_mail(request, doc)
+        e.save()
     if not doc.latest_event(type="changed_last_call_text"):
-        generate_last_call_announcement(request, doc)
-    
+        e = generate_last_call_announcement(request, doc)
+        e.save()
+
     send_last_call_request(request, doc)
 
     e = DocEvent()
@@ -50,8 +52,6 @@ def expire_last_call(doc):
     else:
         raise ValueError("Unexpected document type to expire_last_call(): %s" % doc.type)
 
-    save_document_in_history(doc)
-
     prev_state = doc.get_state(new_state.type_id)
     doc.set_state(new_state)
 
@@ -60,8 +60,7 @@ def expire_last_call(doc):
 
     system = Person.objects.get(name="(System)")
     e = add_state_change_event(doc, system, prev_state, new_state, prev_tags=prev_tags, new_tags=[])
-                    
-    doc.time = (e and e.time) or datetime.datetime.now()
-    doc.save()
+    if e:
+        doc.save_with_history([e])
 
     email_last_call_expired(doc)

ietf/doc/mails.py

@@ -89,6 +89,21 @@ def email_iesg_processing_document(request, doc, changes):
 def html_to_text(html):
     return strip_tags(html.replace("&lt;", "<").replace("&gt;", ">").replace("&amp;", "&").replace("<br>", "\n"))
 
+def email_update_telechat(request, doc, text):
+    (to, cc) = gather_address_lists('doc_telechat_details_changed',doc=doc)
+
+    if not to:
+        return
+    
+    text = strip_tags(text)
+    send_mail(request, to, None,
+              "Telechat update notice: %s" % doc.file_tag(),
+              "doc/mail/update_telechat.txt",
+              dict(text=text,
+                   url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url()),
+              cc=cc)
+
+
 def generate_ballot_writeup(request, doc):
     e = doc.latest_event(type="iana_review")
     iana = e.desc if e else ""
@@ -99,8 +114,8 @@ def generate_ballot_writeup(request, doc):
     e.doc = doc
     e.desc = u"Ballot writeup was generated"
     e.text = unicode(render_to_string("doc/mail/ballot_writeup.txt", {'iana': iana}))
-    e.save()
-    
+
+    # caller is responsible for saving, if necessary
     return e
 
 def generate_ballot_rfceditornote(request, doc):
@@ -154,8 +169,8 @@ def generate_last_call_announcement(request, doc):
     e.doc = doc
     e.desc = u"Last call announcement was generated"
     e.text = unicode(mail)
-    e.save()
 
+    # caller is responsible for saving, if necessary
     return e
 
 
@@ -173,8 +188,8 @@ def generate_approval_mail(request, doc):
     e.doc = doc
     e.desc = u"Ballot approval text was generated"
     e.text = unicode(mail)
-    e.save()
 
+    # caller is responsible for saving, if necessary
     return e
 
 def generate_approval_mail_approved(request, doc):

ietf/doc/migrations/0011_auto_20151027_1127.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+def fix_buggy_author_foreignkey(apps, schema_editor):
+    DocumentAuthor = apps.get_model("doc", "DocumentAuthor")
+    # apparently, we have a buggy key in the DB, fix it
+    DocumentAuthor.objects.filter(author="[<Email: d3e3e3@gmail.com>]").update(author="d3e3e3@gmail.com")
+
+def save_all_documents_in_history(apps, schema_editor):
+    State = apps.get_model("doc", "State")
+    Document = apps.get_model("doc", "Document")
+    DocHistory = apps.get_model("doc", "DocHistory")
+    RelatedDocument = apps.get_model("doc", "RelatedDocument")
+    RelatedDocHistory = apps.get_model("doc", "RelatedDocHistory")
+    DocumentAuthor = apps.get_model("doc", "DocumentAuthor")
+    DocHistoryAuthor = apps.get_model("doc", "DocHistoryAuthor")
+
+    def canonical_name(self):
+        name = self.name
+        state = State.objects.filter(document=self, type_id=self.type_id).first()
+        if self.type_id == "draft" and state.slug == "rfc":
+            a = self.docalias_set.filter(name__startswith="rfc")
+            if a:
+                name = a[0].name
+        elif self.type_id == "charter":
+            return charter_name_for_group(self.chartered_group)
+        return name
+
+    def charter_name_for_group(group):
+        if group.type_id == "rg":
+            top_org = "irtf"
+        else:
+            top_org = "ietf"
+
+        return "charter-%s-%s" % (top_org, group.acronym)
+
+    def save_document_in_history(doc):
+        """Save a snapshot of document and related objects in the database."""
+        def get_model_fields_as_dict(obj):
+            return dict((field.name, getattr(obj, field.name))
+                        for field in obj._meta.fields
+                        if field is not obj._meta.pk)
+
+        # copy fields
+        fields = get_model_fields_as_dict(doc)
+        fields["doc"] = doc
+        fields["name"] = canonical_name(doc)
+
+        dochist = DocHistory(**fields)
+        dochist.save()
+
+        # copy many to many
+        for field in doc._meta.many_to_many:
+            if field.rel.through and field.rel.through._meta.auto_created:
+                setattr(dochist, field.name, getattr(doc, field.name).all())
+
+        # copy remaining tricky many to many
+        def transfer_fields(obj, HistModel):
+            mfields = get_model_fields_as_dict(item)
+            # map doc -> dochist
+            for k, v in mfields.iteritems():
+                if v == doc:
+                    mfields[k] = dochist
+            HistModel.objects.create(**mfields)
+
+        for item in RelatedDocument.objects.filter(source=doc):
+            transfer_fields(item, RelatedDocHistory)
+
+        for item in DocumentAuthor.objects.filter(document=doc):
+            transfer_fields(item, DocHistoryAuthor)
+
+        return dochist
+
+    from django.conf import settings
+    settings.DEBUG = False # prevent out-of-memory problems
+
+    for d in Document.objects.iterator():
+        save_document_in_history(d)
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('doc', '0010_auto_20150930_0251'),
+        ('group', '0007_auto_20150930_0758'),
+    ]
+
+    operations = [
+        migrations.RunPython(fix_buggy_author_foreignkey),
+        migrations.RunPython(save_all_documents_in_history)
+    ]