Merged from log:branch/iesg-tracker@2571: IOLA's port of cron-scripts.

 - Legacy-Id: 2578

Author: levkowetz

ietf/announcements/models.py

@@ -74,7 +74,7 @@ class ScheduledAnnouncement(models.Model):
     cc_val = models.TextField(blank=True,null=True)
     body = models.TextField(blank=True)
     actual_sent_date = models.DateField(null=True, blank=True)
-    actual_sent_time = models.CharField(blank=True, max_length=50)
+    actual_sent_time = models.CharField(blank=True, max_length=50) # should be time, but database contains oddities
     first_q = models.IntegerField(null=True, blank=True)
     second_q = models.IntegerField(null=True, blank=True)
     note = models.TextField(blank=True,null=True)

ietf/announcements/send_scheduled.py

@@ -0,0 +1,32 @@
+import re, datetime, email
+
+from ietf.utils.mail import send_mail_text, send_mail_mime
+
+first_dot_on_line_re = re.compile(r'^\.', re.MULTILINE)
+
+def send_scheduled_announcement(announcement):
+    # for some reason, the old Perl code base substituted away . on line starts
+    body = first_dot_on_line_re.sub("", announcement.body)
+    
+    announcement.content_type
+    extra = {}
+    if announcement.replyto:
+        extra['Reply-To'] = announcement.replyto
+
+    content_type = announcement.content_type.lower()
+    if not content_type or 'text/plain' in content_type:
+        send_mail_text(None, announcement.to_val, announcement.from_val, announcement.subject,
+                       body, cc=announcement.cc_val, bcc=announcement.bcc_val)
+    elif 'multipart' in content_type:
+        # make body a real message so we can parse it
+        body = ("MIME-Version: 1.0\r\nContent-Type: %s\r\n" % content_type) + body
+        
+        msg = email.message_from_string(body.encode("utf-8"))
+        send_mail_mime(None, announcement.to_val, announcement.from_val, announcement.subject,
+                       msg, cc=announcement.cc_val, bcc=announcement.bcc_val)
+
+    now = datetime.datetime.now()
+    announcement.actual_sent_date = now.date()
+    announcement.actual_sent_time = str(now.time())
+    announcement.mail_sent = True
+    announcement.save()

ietf/announcements/tests.py

@@ -1,5 +1,9 @@
+import django.test
 
 from ietf.utils.test_utils import SimpleUrlTestCase, canonicalize_sitemap
+from ietf.utils.test_runner import mail_outbox
+
+from ietf.announcements.models import ScheduledAnnouncement
 
 class AnnouncementsUrlTestCase(SimpleUrlTestCase):
     def testUrls(self):
@@ -10,3 +14,46 @@ def doCanonicalize(self, url, content):
         else:
             return content
 
+class SendScheduledAnnouncementsTestCase(django.test.TestCase):
+    def test_send_plain_announcement(self):
+        a = ScheduledAnnouncement.objects.create(
+            mail_sent=False,
+            subject="This is a test",
+            to_val="test@example.com",
+            from_val="testmonkey@example.com",
+            cc_val="cc.a@example.com, cc.b@example.com",
+            bcc_val="bcc@example.com",
+            body="Hello World!",
+            content_type="",
+            )
+
+        mailbox_before = len(mail_outbox)
+        
+        from ietf.announcements.send_scheduled import send_scheduled_announcement
+        send_scheduled_announcement(a)
+
+        self.assertEquals(len(mail_outbox), mailbox_before + 1)
+        self.assertTrue("This is a test" in mail_outbox[-1]["Subject"])
+        self.assertTrue(ScheduledAnnouncement.objects.get(id=a.id).mail_sent)
+
+    def test_send_mime_announcement(self):
+        a = ScheduledAnnouncement.objects.create(
+            mail_sent=False,
+            subject="This is a test",
+            to_val="test@example.com",
+            from_val="testmonkey@example.com",
+            cc_val="cc.a@example.com, cc.b@example.com",
+            bcc_val="bcc@example.com",
+            body='--NextPart\r\n\r\nA New Internet-Draft is available from the on-line Internet-Drafts directories.\r\n--NextPart\r\nContent-Type: Message/External-body;\r\n\tname="draft-huang-behave-bih-01.txt";\r\n\tsite="ftp.ietf.org";\r\n\taccess-type="anon-ftp";\r\n\tdirectory="internet-drafts"\r\n\r\nContent-Type: text/plain\r\nContent-ID:     <2010-07-30001541.I-D@ietf.org>\r\n\r\n--NextPart--',
+            content_type='Multipart/Mixed; Boundary="NextPart"',
+            )
+
+        mailbox_before = len(mail_outbox)
+        
+        from ietf.announcements.send_scheduled import send_scheduled_announcement
+        send_scheduled_announcement(a)
+
+        self.assertEquals(len(mail_outbox), mailbox_before + 1)
+        self.assertTrue("This is a test" in mail_outbox[-1]["Subject"])
+        self.assertTrue("--NextPart" in mail_outbox[-1].as_string())
+        self.assertTrue(ScheduledAnnouncement.objects.get(id=a.id).mail_sent)

ietf/bin/expire-ids

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+import datetime, os
+import syslog
+
+from ietf import settings
+from django.core import management
+management.setup_environ(settings)
+
+syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_LOCAL0)
+
+from ietf.idrfc.expire import *
+
+if not in_id_expire_freeze():
+    for doc in get_expired_ids():
+        send_expire_notice_for_id(doc)
+        expire_id(doc)
+        syslog.syslog("Expired %s (id=%s)%s" % (doc.file_tag(), doc.id_document_tag, " in the ID Tracker" if doc.idinternal else ""))
+
+clean_up_id_files()

ietf/bin/expire-last-calls

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+
+import datetime, os
+import syslog
+
+from ietf import settings
+from django.core import management
+management.setup_environ(settings)
+
+syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_LOCAL0)
+
+from ietf.idrfc.lastcall import *
+
+drafts = get_expired_last_calls()
+for doc in drafts:
+    expire_last_call(doc)
+    syslog.syslog("Expired last call for %s (id=%s)" % (doc.file_tag(), doc.id_document_tag))

ietf/bin/send-scheduled-mail

@@ -1,33 +1,42 @@
 #!/usr/bin/env python
-#
-from django.core.management import setup_environ
+
+import datetime, os, sys
+import syslog
+
 from ietf import settings
+from django.core import management
+management.setup_environ(settings)
+
+syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_LOCAL0)
 
-setup_environ(settings)
 
-from django.db.models import Q
-from django.db import connection
 from ietf.announcements.models import ScheduledAnnouncement
-from ietf.utils.mail import send_mail_text
-import datetime
+from ietf.announcements.send_scheduled import *
+from django.db.models import Q
+
+if not len(sys.argv) == 2 or sys.argv[1] not in ('all', 'rsync', 'specific'):
+    print "USAGE: %s <all | rsync | specific>" % os.path.basename(__file__)
+    print "'all' means all not sent"
+    print "'rsync' means all not sent where to-be-sent-date is null"
+    print "'specific' means all not sent that are due to be sent"
+    sys.exit(1)
+
+mode = sys.argv[1]
 
 now = datetime.datetime.now()
-candidates = ScheduledAnnouncement.objects.filter(to_be_sent_date__isnull=True) | ScheduledAnnouncement.objects.exclude(to_be_sent_date__gt=now.date())
-candidates = candidates.filter(mail_sent=False) #.exclude(to_be_sent_date__gt=now.date())
-for c in candidates:
-    if c.to_be_sent_time:
-        tbst = datetime.time( *[int(t) for t in c.to_be_sent_time.split(":")] )
-        if tbst > now.time():
-	    # To be sent today, but not yet.
-	    continue
-    extra={}
-    if c.content_type:
-        extra['Content-Type'] = c.content_type
-    if c.replyto:
-        extra['Reply-To'] = c.replyto
-    send_mail_text(None, to=c.to_val, frm=c.from_val, subject=c.subject,
-	txt=c.body, cc=c.cc_val, extra=extra, bcc=c.bcc_val)
-    c.actual_sent_date=now.date()
-    c.actual_sent_time=str(now.time()) # sigh
-    c.mail_sent=True
-    c.save()
+now = datetime.datetime(2010, 8, 5)
+
+announcements = ScheduledAnnouncement.objects.filter(mail_sent=False)
+if mode == "rsync":
+    # include bogus 0000-00-00 entries
+    announcements = announcements.filter(Q(to_be_sent_date=None) | Q(to_be_sent_date__lte=datetime.date.min))
+elif mode == "specific":
+    # exclude null/bogus entries
+    announcements = announcements.exclude(Q(to_be_sent_date=None) | Q(to_be_sent_date__lte=datetime.date.min))
+    announcements = announcements.filter(to_be_sent_date__lte=now.date(),
+                                         to_be_sent_time__lte=now.time())
+    
+for announcement in announcements:
+    send_scheduled_announcement(announcement)
+    
+    syslog.syslog('Sent scheduled announcement %s "%s"' % (announcement.id, announcement.subject))

ietf/idrfc/expire.py

@@ -0,0 +1,122 @@
+# expiry of Internet Drafts
+
+from django.conf import settings
+from django.template.loader import render_to_string
+from django.db.models import Q
+
+import datetime, os, shutil, glob, re
+
+from ietf.idtracker.models import InternetDraft, IDDates, IDStatus, IDState, DocumentComment
+from ietf.utils.mail import send_mail
+from ietf.idrfc.utils import log_state_changed, add_document_comment
+
+def in_id_expire_freeze(when=None):
+    if when == None:
+        when = datetime.datetime.now()
+        
+    d = IDDates.objects.get(id=IDDates.SECOND_CUT_OFF).date
+    # for some reason, the old Perl code started at 9 am
+    second_cut_off = datetime.datetime.combine(d, datetime.time(9, 0))
+    
+    d = IDDates.objects.get(id=IDDates.IETF_MONDAY).date
+    ietf_monday = datetime.datetime.combine(d, datetime.time(0, 0))
+    
+    return second_cut_off <= when < ietf_monday
+
+def get_expired_ids():
+    cut_off = datetime.date.today() - datetime.timedelta(days=InternetDraft.DAYS_TO_EXPIRE)
+
+    return InternetDraft.objects.filter(
+        revision_date__lte=cut_off,
+        status__status="Active",
+        review_by_rfc_editor=0).filter(
+        Q(idinternal=None) | Q(idinternal__cur_state__document_state_id__gte=42))
+
+def send_expire_notice_for_id(doc):
+    doc.dunn_sent_date = datetime.date.today()
+    doc.save()
+
+    if not doc.idinternal:
+        return
+    
+    request = None
+    to = u"%s <%s>" % doc.idinternal.job_owner.person.email()
+    send_mail(request, to,
+              "I-D Expiring System <ietf-secretariat-reply@ietf.org>",
+              u"I-D was expired %s" % doc.file_tag(),
+              "idrfc/id_expired_email.txt",
+              dict(doc=doc))
+
+def expire_id(doc):
+    def move_file(f):
+        src = os.path.join(settings.INTERNET_DRAFT_PATH, f)
+        dst = os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, f)
+
+        if os.path.exists(src):
+            shutil.move(src, dst)
+
+    move_file("%s-%s.txt" % (doc.filename, doc.revision_display()))
+    move_file("%s-%s.ps" % (doc.filename, doc.revision_display()))
+    move_file("%s-%s.pdf" % (doc.filename, doc.revision_display()))
+
+    new_revision = "%02d" % (int(doc.revision) + 1)
+
+    new_file = open(os.path.join(settings.INTERNET_DRAFT_PATH, "%s-%s.txt" % (doc.filename, new_revision)), 'w')
+    txt = render_to_string("idrfc/expire_text.txt",
+                           dict(doc=doc,
+                                authors=[a.person.email() for a in doc.authors.all()],
+                                expire_days=InternetDraft.DAYS_TO_EXPIRE))          
+    new_file.write(txt)
+    new_file.close()
+    
+    doc.revision = new_revision
+    doc.expiration_date = datetime.date.today()
+    doc.last_modified_date = datetime.date.today()
+    doc.status = IDStatus.objects.get(status="Expired")
+    doc.save()
+
+    if doc.idinternal:
+        if doc.idinternal.cur_state_id != IDState.DEAD:
+            doc.idinternal.change_state(IDState.objects.get(document_state_id=IDState.DEAD), None)
+            log_state_changed(None, doc, "system")
+
+        add_document_comment(None, doc, "Document is expired by system")
+
+def clean_up_id_files():
+    """Move unidentified and old files out of the Internet Draft directory."""
+    cut_off = datetime.date.today() - datetime.timedelta(days=InternetDraft.DAYS_TO_EXPIRE)
+
+    pattern = os.path.join(settings.INTERNET_DRAFT_PATH, "draft-*.*")
+    files = []
+    filename_re = re.compile('^(.*)-(\d+)$')
+    for path in glob.glob(pattern):
+        basename = os.path.basename(path)
+        stem, ext = os.path.splitext(basename)
+        match = filename_re.search(stem)
+        if not match:
+            filename, revision = ("UNKNOWN", "00")
+        else:
+            filename, revision = match.groups()
+
+        def move_file_to(subdir):
+            shutil.move(path,
+                        os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, subdir, basename))
+            
+        try:
+            doc = InternetDraft.objects.get(filename=filename, revision=revision)
+
+            if doc.status_id == 3:
+                if ext != ".txt":
+                    move_file_to("unknown_ids")
+            elif doc.status_id in (2, 4, 5, 6) and doc.expiration_date and doc.expiration_date < cut_off:
+                if os.path.getsize(path) < 1500:
+                    move_file_to("deleted_tombstones")
+                    # revert version after having deleted tombstone
+                    doc.revision = "%02d" % (int(revision) - 1)
+                    doc.expired_tombstone = True
+                    doc.save()
+                else:
+                    move_file_to("expired_without_tombstone")
+            
+        except InternetDraft.DoesNotExist:
+            move_file_to("unknown_ids")

ietf/idrfc/fixtures/base.xml

@@ -1283,4 +1283,34 @@ withdrawn, etc.)</field>
         <field type="CharField" name="name">Individual Submissions</field>
         <field type="CharField" name="name_key">INDIVIDUAL SUBMISSIONS</field>
     </object>
+    <object pk="1" model="idtracker.iddates">
+        <field type="DateField" name="date">2010-07-05</field>
+        <field type="CharField" name="description">1st Cut-Off Date (Initial version)</field>
+        <field type="CharField" name="f_name">first</field>
+    </object>
+    <object pk="2" model="idtracker.iddates">
+        <field type="DateField" name="date">2010-07-12</field>
+        <field type="CharField" name="description">2nd Cut-Off Date (Update version)</field>
+        <field type="CharField" name="f_name">second</field>
+    </object>
+    <object pk="3" model="idtracker.iddates">
+        <field type="DateField" name="date">2010-07-26</field>
+        <field type="CharField" name="description">Date of Monday of IETF</field>
+        <field type="CharField" name="f_name">third</field>
+    </object>
+    <object pk="4" model="idtracker.iddates">
+        <field type="DateField" name="date">2010-07-29</field>
+        <field type="CharField" name="description">All I-Ds will be processed by</field>
+        <field type="CharField" name="f_name">fourth</field>
+    </object>
+    <object pk="5" model="idtracker.iddates">
+        <field type="DateField" name="date">2010-08-02</field>
+        <field type="CharField" name="description">Date of Monday after IETF</field>
+        <field type="CharField" name="f_name">fifth</field>
+    </object>
+    <object pk="6" model="idtracker.iddates">
+        <field type="DateField" name="date">2010-06-28</field>
+        <field type="CharField" name="description">Date for List of Approved V-00 Submissions from WG Chairs</field>
+        <field type="CharField" name="f_name">sixth</field>
+    </object>
 </django-objects>

ietf/idrfc/generate_fixtures.py

@@ -45,6 +45,7 @@ def output(name, qs):
 base.extend(WGType.objects.all())
 base.extend(TelechatDates.objects.all())
 base.extend(Acronym.objects.filter(acronym_id=Acronym.INDIVIDUAL_SUBMITTER))
+base.extend(IDDates.objects.all())
 
 output("base", base)