Merged branch/iola/statesync up to @5118 to trunk.

 - Legacy-Id: 5147

Author: levkowetz

ietf/bin/email-sync-discrepancies

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+import sys, os
+import syslog
+
+# boilerplate
+basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+sys.path = [ basedir ] + sys.path
+
+from ietf import settings
+from django.core import management
+management.setup_environ(settings)
+
+
+from optparse import OptionParser
+
+parser = OptionParser()
+parser.add_option("-t", "--to", dest="to",
+                  help="Email address to send report to", metavar="EMAIL")
+
+options, args = parser.parse_args()
+
+
+syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_LOCAL0)
+
+from ietf.sync.mails import email_discrepancies
+
+receivers = ["iesg-secretary@ietf.org"]
+
+if options.to:
+    receivers = [options.to]
+
+email_discrepancies(receivers)
+
+syslog.syslog("Emailed sync discrepancies to %s" % receivers)

ietf/bin/iana-changes-updates

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+import os, sys, re, json, datetime, optparse
+import syslog
+
+# boilerplate
+basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+sys.path = [ basedir ] + sys.path
+
+from ietf import settings
+from django.core import management
+management.setup_environ(settings)
+
+
+from optparse import OptionParser
+
+parser = OptionParser()
+parser.add_option("-f", "--from", dest="start",
+                  help="Start time, defaults to a little less than 23 hours ago", metavar="YYYY-MM-DD HH:MM:SS")
+parser.add_option("-t", "--to", dest="end",
+                  help="End time, defaults to 23 hours later than from", metavar="YYYY-MM-DD HH:MM:SS")
+parser.add_option("", "--no-email", dest="send_email", default=True, action="store_false",
+                  help="Skip sending emails")
+
+options, args = parser.parse_args()
+
+# compensate to avoid we ask for something that happened now and then
+# don't get it back because our request interval is slightly off
+CLOCK_SKEW_COMPENSATION = 5 # seconds
+
+# actually the interface accepts 24 hours, but then we get into
+# trouble with daylights savings - meh
+MAX_INTERVAL_ACCEPTED_BY_IANA = datetime.timedelta(hours=23)
+
+
+start = datetime.datetime.now() - datetime.timedelta(hours=23) + datetime.timedelta(seconds=CLOCK_SKEW_COMPENSATION)
+if options.start:
+    start = datetime.datetime.strptime(options.start, "%Y-%m-%d %H:%M:%S")
+
+end = start + datetime.timedelta(hours=23)
+if options.end:
+    end = datetime.datetime.strptime(options.end, "%Y-%m-%d %H:%M:%S")
+
+syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_LOCAL0)
+
+
+from ietf.sync.iana import *
+
+syslog.syslog("Updating history log with new changes from IANA from %s, period %s - %s" % (CHANGES_URL, start, end))
+
+t = start
+while t < end:
+    # the IANA server doesn't allow us to fetch more than a certain
+    # period, so loop over the requested period and make multiple
+    # requests if necessary
+
+    text = fetch_changes_json(CHANGES_URL, t, min(end, t + MAX_INTERVAL_ACCEPTED_BY_IANA))
+    changes = parse_changes_json(text)
+    added_events, warnings = update_history_with_changes(changes, send_email=options.send_email)
+
+    for e in added_events:
+        syslog.syslog("Added event for %s %s: %s" % (e.doc_id, e.time, e.desc))
+
+    for w in warnings:
+        syslog.syslog("WARNING: %s" % w)
+
+    t += MAX_INTERVAL_ACCEPTED_BY_IANA

ietf/bin/iana-protocols-updates

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+
+import os, sys, re, json, datetime
+import syslog
+
+syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_LOCAL0)
+
+# boilerplate
+basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+sys.path = [ basedir ] + sys.path
+
+from ietf import settings
+from django.core import management
+management.setup_environ(settings)
+
+
+from ietf.sync.iana import *
+
+def chunks(l, n):
+    """Split list l up in chunks of max size n."""
+    return (l[i:i+n] for i in xrange(0, len(l), n))
+
+syslog.syslog("Updating history log with new RFC entries from IANA protocols page %s" % PROTOCOLS_URL)
+
+# FIXME: this needs to be the date where this tool is first deployed
+rfc_must_published_later_than = datetime.datetime(2012, 11, 26, 0, 0, 0)
+
+text = fetch_protocol_page(PROTOCOLS_URL)
+rfc_numbers = parse_protocol_page(text)
+for chunk in chunks(rfc_numbers, 100):
+    updated = update_rfc_log_from_protocol_page(chunk, rfc_must_published_later_than)
+
+    for d in updated:
+        syslog.syslog("Added history entry for %s" % d.display_name())

ietf/bin/iana-review-email

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+import os, sys, re, json, datetime, optparse
+import syslog
+
+# boilerplate
+basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+sys.path = [ basedir ] + sys.path
+
+from ietf import settings
+from django.core import management
+management.setup_environ(settings)
+
+
+syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_LOCAL0)
+
+from ietf.sync.iana import *
+from ietf.doc.models import Document
+
+msg = sys.stdin.read()
+
+doc_name, review_time, by, comment = parse_review_email(msg)
+
+syslog.syslog(u"Read IANA review email for %s at %s by %s" % (doc_name, review_time, by))
+
+if by.name == "(System)":
+    syslog.syslog("WARNING: person responsible for email does not have a IANA role")
+
+try:
+    add_review_comment(doc_name, review_time, by, comment)
+except Document.DoesNotExist:
+    syslog.syslog("ERROR: unknown document %s" % doc_name)
+

ietf/bin/rfc-editor-index-updates

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+
+import os, sys, re, json, datetime
+import syslog
+
+syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_LOCAL0)
+
+# boilerplate
+basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+sys.path = [ basedir ] + sys.path
+
+from ietf import settings
+from django.core import management
+management.setup_environ(settings)
+
+
+from optparse import OptionParser
+
+parser = OptionParser()
+parser.add_option("-d", dest="skip_date",
+                  help="To speed up processing skip RFCs published before this date (default is one year ago)", metavar="YYYY-MM-DD")
+
+options, args = parser.parse_args()
+
+skip_date = datetime.date.today() - datetime.timedelta(days=365)
+if options.skip_date:
+    skip_date = datetime.datetime.strptime(options.skip_date, "%Y-%m-%d").date()
+
+from ietf.sync.rfceditor import *
+
+syslog.syslog("Updating document metadata from RFC index from %s" % QUEUE_URL)
+
+response = fetch_index_xml(INDEX_URL)
+data = parse_index(response)
+
+if len(data) < MIN_INDEX_RESULTS:
+    syslog.syslog("Not enough results, only %s" % len(data))
+    sys.exit(1)
+
+changed = update_docs_from_rfc_index(data, skip_older_than_date=skip_date)
+for c in changed:
+    syslog.syslog(c)

ietf/bin/rfc-editor-queue-updates

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+import os, sys, re, json, datetime
+import syslog
+
+syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_LOCAL0)
+
+# boilerplate
+basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+sys.path = [ basedir ] + sys.path
+
+from ietf import settings
+from django.core import management
+management.setup_environ(settings)
+
+
+from ietf.sync.rfceditor import *
+
+syslog.syslog("Updating RFC Editor queue states from %s" % QUEUE_URL)
+
+response = fetch_queue_xml(QUEUE_URL)
+drafts, warnings = parse_queue(response)
+for w in warnings:
+    syslog.syslog(u"WARNING: %s" % w)
+
+if len(drafts) < MIN_QUEUE_RESULTS:
+    syslog.syslog("Not enough results, only %s" % len(drafts))
+    sys.exit(1)
+
+changed, warnings = update_drafts_from_queue(drafts)
+for w in warnings:
+    syslog.syslog(u"WARNING: %s" % w)
+
+for c in changed:
+    syslog.syslog(u"Updated %s" % c)

ietf/doc/admin.py

@@ -142,6 +142,8 @@ def rev(self, obj):
 admin.site.register(DocEvent, DocEventAdmin)
 
 admin.site.register(NewRevisionDocEvent, DocEventAdmin)
+admin.site.register(StateDocEvent, DocEventAdmin)
+admin.site.register(ConsensusDocEvent, DocEventAdmin)
 admin.site.register(BallotDocEvent, DocEventAdmin)
 admin.site.register(WriteupDocEvent, DocEventAdmin)
 admin.site.register(LastCallDocEvent, DocEventAdmin)