Port send queue script and tests and import ScheduledAnnouncement + some cleanups

 - Legacy-Id: 3163

Author: OleLaursen

ietf/announcements/admin.py

@@ -26,7 +26,7 @@ class ScheduledAnnouncementAdmin(admin.ModelAdmin):
 if settings.USE_DB_REDESIGN_PROXY_CLASSES:
     class MessageAdmin(admin.ModelAdmin):
         list_display = ["time", "by", "subject", "groups"]
-        search_fields = ["text"]
+        search_fields = ["body"]
         raw_id_fields = ["by"]
 
         def groups(self, instance):
@@ -37,7 +37,7 @@ def groups(self, instance):
     class SendQueueAdmin(admin.ModelAdmin):
         list_display = ["time", "by", "message", "send_at", "sent_at"]
         list_filter = ["time", "send_at", "sent_at"]
-        search_fields = ["message__text"]
+        search_fields = ["message__body"]
         raw_id_fields = ["by"]
 
     admin.site.register(SendQueue, SendQueueAdmin)

ietf/announcements/models.py

@@ -100,11 +100,12 @@ class Message(models.Model):
 
         subject = models.CharField(max_length=255)
         frm = models.CharField(max_length=255)
-        to = models.CharField(max_length=255)
-        cc = models.CharField(max_length=255, blank=True)
+        to = models.CharField(max_length=1024)
+        cc = models.CharField(max_length=1024, blank=True)
         bcc = models.CharField(max_length=255, blank=True)
         reply_to = models.CharField(max_length=255, blank=True)
-        text = models.TextField()
+        body = models.TextField()
+        content_type = models.CharField(max_length=255, blank=True)
 
         related_groups = models.ManyToManyField(Group, blank=True)
 
@@ -118,10 +119,12 @@ class SendQueue(models.Model):
         time = models.DateTimeField(default=datetime.datetime.now)
         by = models.ForeignKey(Person)
 
-        comment = models.TextField()
         message = models.ForeignKey(Message)
+        
         send_at = models.DateTimeField(blank=True, null=True)
         sent_at = models.DateTimeField(blank=True, null=True)
 
+        note = models.TextField(blank=True)
+        
         class Meta:
             ordering = ['time']

ietf/announcements/send_scheduled.py

@@ -1,5 +1,7 @@
 import re, datetime, email
 
+from django.conf import settings
+
 from ietf.utils.mail import send_mail_text, send_mail_mime
 
 first_dot_on_line_re = re.compile(r'^\.', re.MULTILINE)
@@ -8,7 +10,6 @@ def send_scheduled_announcement(announcement):
     # for some reason, the old Perl code base substituted away . on line starts
     body = first_dot_on_line_re.sub("", announcement.body)
 
-    announcement.content_type
     extra = {}
     if announcement.replyto:
         extra['Reply-To'] = announcement.replyto
@@ -30,3 +31,32 @@ def send_scheduled_announcement(announcement):
     announcement.actual_sent_time = str(now.time())
     announcement.mail_sent = True
     announcement.save()
+
+
+def send_scheduled_announcementREDESIGN(send_queue):
+    message = send_queue.message
+    
+    # for some reason, the old Perl code base substituted away . on line starts
+    body = first_dot_on_line_re.sub("", message.body)
+    
+    extra = {}
+    if message.reply_to:
+        extra['Reply-To'] = message.reply_to
+
+    content_type = message.content_type.lower()
+    if not content_type or 'text/plain' in content_type:
+        send_mail_text(None, message.to, message.frm, message.subject,
+                       body, cc=message.cc, bcc=message.bcc)
+    elif 'multipart' in content_type:
+        # make body a real message so we can parse it
+        body = ("MIME-Version: 1.0\r\nContent-Type: %s\r\n" % content_type) + body
+        
+        msg = email.message_from_string(body.encode("utf-8"))
+        send_mail_mime(None, message.to, message.frm, message.subject,
+                       msg, cc=message.cc, bcc=message.bcc)
+
+    send_queue.sent_at = datetime.datetime.now()
+    send_queue.save()
+
+if settings.USE_DB_REDESIGN_PROXY_CLASSES:
+    send_scheduled_announcement = send_scheduled_announcementREDESIGN

ietf/announcements/tests.py

@@ -1,7 +1,11 @@
+import datetime
+
+from django.conf import settings
 import django.test
 
 from ietf.utils.test_utils import SimpleUrlTestCase, canonicalize_sitemap
 from ietf.utils.test_runner import mail_outbox
+from ietf.utils.test_data import make_test_data
 
 from ietf.announcements.models import ScheduledAnnouncement
 
@@ -57,3 +61,73 @@ def test_send_mime_announcement(self):
         self.assertTrue("This is a test" in mail_outbox[-1]["Subject"])
         self.assertTrue("--NextPart" in mail_outbox[-1].as_string())
         self.assertTrue(ScheduledAnnouncement.objects.get(id=a.id).mail_sent)
+
+
+class SendScheduledAnnouncementsTestCaseREDESIGN(django.test.TestCase):
+    def test_send_plain_announcement(self):
+        from ietf.announcements.models import Message, SendQueue
+        from redesign.person.models import Person
+        
+        make_test_data()
+
+        msg = Message.objects.create(
+            by=Person.objects.get(name="(System)"),
+            subject="This is a test",
+            to="test@example.com",
+            frm="testmonkey@example.com",
+            cc="cc.a@example.com, cc.b@example.com",
+            bcc="bcc@example.com",
+            body="Hello World!",
+            content_type="",
+            )
+
+        q = SendQueue.objects.create(
+            by=Person.objects.get(name="(System)"),
+            message=msg,
+            send_at=datetime.datetime.now() + datetime.timedelta(hours=12)
+            )
+
+        mailbox_before = len(mail_outbox)
+        
+        from ietf.announcements.send_scheduled import send_scheduled_announcement
+        send_scheduled_announcement(q)
+
+        self.assertEquals(len(mail_outbox), mailbox_before + 1)
+        self.assertTrue("This is a test" in mail_outbox[-1]["Subject"])
+        self.assertTrue(SendQueue.objects.get(id=q.id).sent_at)
+
+    def test_send_mime_announcement(self):
+        from ietf.announcements.models import Message, SendQueue
+        from redesign.person.models import Person
+        
+        make_test_data()
+
+        msg = Message.objects.create(
+            by=Person.objects.get(name="(System)"),
+            subject="This is a test",
+            to="test@example.com",
+            frm="testmonkey@example.com",
+            cc="cc.a@example.com, cc.b@example.com",
+            bcc="bcc@example.com",
+            body='--NextPart\r\n\r\nA New Internet-Draft is available from the on-line Internet-Drafts directories.\r\n--NextPart\r\nContent-Type: Message/External-body;\r\n\tname="draft-huang-behave-bih-01.txt";\r\n\tsite="ftp.ietf.org";\r\n\taccess-type="anon-ftp";\r\n\tdirectory="internet-drafts"\r\n\r\nContent-Type: text/plain\r\nContent-ID:     <2010-07-30001541.I-D@ietf.org>\r\n\r\n--NextPart--',
+            content_type='Multipart/Mixed; Boundary="NextPart"',
+            )
+
+        q = SendQueue.objects.create(
+            by=Person.objects.get(name="(System)"),
+            message=msg,
+            send_at=datetime.datetime.now() + datetime.timedelta(hours=12)
+            )
+        
+        mailbox_before = len(mail_outbox)
+        
+        from ietf.announcements.send_scheduled import send_scheduled_announcement
+        send_scheduled_announcement(q)
+
+        self.assertEquals(len(mail_outbox), mailbox_before + 1)
+        self.assertTrue("This is a test" in mail_outbox[-1]["Subject"])
+        self.assertTrue("--NextPart" in mail_outbox[-1].as_string())
+        self.assertTrue(SendQueue.objects.get(id=q.id).sent_at)
+
+if settings.USE_DB_REDESIGN_PROXY_CLASSES:
+    SendScheduledAnnouncementsTestCase = SendScheduledAnnouncementsTestCaseREDESIGN

ietf/bin/send-scheduled-mail

@@ -14,7 +14,7 @@ from ietf.announcements.models import ScheduledAnnouncement
 from ietf.announcements.send_scheduled import *
 from django.db.models import Q
 
-if not len(sys.argv) == 2 or sys.argv[1] not in ('all', 'rsync', 'specific'):
+if len(sys.argv) != 2 or sys.argv[1] not in ('all', 'rsync', 'specific'):
     print "USAGE: %s <all | rsync | specific>" % os.path.basename(__file__)
     print "'all' means all not sent"
     print "'rsync' means all not sent where to-be-sent-date is null"
@@ -24,19 +24,28 @@ if not len(sys.argv) == 2 or sys.argv[1] not in ('all', 'rsync', 'specific'):
 mode = sys.argv[1]
 
 now = datetime.datetime.now()
-now = datetime.datetime(2010, 8, 5)
-
-announcements = ScheduledAnnouncement.objects.filter(mail_sent=False)
-if mode == "rsync":
-    # include bogus 0000-00-00 entries
-    announcements = announcements.filter(Q(to_be_sent_date=None) | Q(to_be_sent_date__lte=datetime.date.min))
-elif mode == "specific":
-    # exclude null/bogus entries
-    announcements = announcements.exclude(Q(to_be_sent_date=None) | Q(to_be_sent_date__lte=datetime.date.min))
-    announcements = announcements.filter(to_be_sent_date__lte=now.date(),
-                                         to_be_sent_time__lte=now.time())
+
+if settings.USE_DB_REDESIGN_PROXY_CLASSES:
+    from ietf.announcements.models import SendQueue
+    announcements = SendQueue.objects.filter(sent_at=None)
+    if mode == "rsync":
+        announcements = announcements.filter(send_at=None)
+    elif mode == "specific":
+        announcements = announcements.exclude(send_at=None).filter(send_at__lte=now)
+else:
+    announcements = ScheduledAnnouncement.objects.filter(mail_sent=False)
+    if mode == "rsync":
+        # include bogus 0000-00-00 entries
+        announcements = announcements.filter(Q(to_be_sent_date=None) | Q(to_be_sent_date__lte=datetime.date.min))
+    elif mode == "specific":
+        # exclude null/bogus entries
+        announcements = announcements.exclude(Q(to_be_sent_date=None) | Q(to_be_sent_date__lte=datetime.date.min))
+
+        announcements = announcements.filter(to_be_sent_date__lte=now.date(),
+                                             to_be_sent_time__lte=now.time())
 
 for announcement in announcements:
     send_scheduled_announcement(announcement)
-    
-    syslog.syslog('Sent scheduled announcement %s "%s"' % (announcement.id, announcement.subject))
+
+    subject = announcement.message.subject if settings.USE_DB_REDESIGN_PROXY_CLASSES else announcement.subject
+    syslog.syslog(u'Sent scheduled announcement %s "%s"' % (announcement.id, subject))

ietf/idrfc/views_ballot.py

@@ -292,7 +292,6 @@ def edit_positionREDESIGN(request, name):
                 doc.time = pos.time
                 doc.save()
 
-            # FIXME: test
             if request.POST.get("send_mail"):
                 qstr = "?return_to_url=%s" % return_to_url
                 if request.GET.get('ad'):

ietf/templates/announcements/message_detail.html

@@ -13,6 +13,6 @@ <h1>NomCom Message</h1>
 </p>
 <hr width="400" align="left" />
 <pre>
-{{ message.text|escape }}
+{{ message.body|escape }}
 </pre>
 {% endblock %}

redesign/doc/models.py

@@ -185,21 +185,6 @@ class Meta:
         verbose_name_plural = "document aliases"
 
 
-# class Ballot(models.Model):             # A collection of ballot positions
-#     """A collection of ballot positions, and the actions taken during the
-#     lifetime of the ballot.
-
-#     The actual ballot positions are found by searching Messages for
-#     BallotPositions for this document between the dates indicated by
-#     self.initiated.time and (self.closed.time or now)
-#     """
-#     initiated = models.ForeignKey(Message,                        related_name="initiated_ballots")
-#     deferred  = models.ForeignKey(Message, null=True, blank=True, related_name="deferred_ballots")
-#     last_call = models.ForeignKey(Message, null=True, blank=True, related_name="lastcalled_ballots")
-#     closed    = models.ForeignKey(Message, null=True, blank=True, related_name="closed_ballots")
-#     announced = models.ForeignKey(Message, null=True, blank=True, related_name="announced_ballots")
-
-
 EVENT_TYPES = [
     # core events
     ("new_revision", "Added new revision"),

redesign/importing/import-announcements.py

@@ -16,21 +16,21 @@
 from redesign.group.models import *
 from redesign.name.utils import name
 from redesign.importing.utils import old_person_to_person
-from ietf.announcements.models import Message
-from ietf.announcements.models import Announcement, PersonOrOrgInfo, AnnouncedTo, AnnouncedFrom
+from ietf.announcements.models import Message, SendQueue
+from ietf.announcements.models import Announcement, PersonOrOrgInfo, AnnouncedTo, AnnouncedFrom, ScheduledAnnouncement
+from ietf.idtracker.models import IESGLogin
 
 # assumptions:
 #  - nomcom groups have been imported
-#  - persons have been imported
+#  - persons have been imported (Announcement originators and IESGLogins)
 
-# imports Announcements
-
-# FIXME: should import ScheduledAnnouncements
+# imports Announcement, ScheduledAnnouncement
 
 system = Person.objects.get(name="(System)")
 
 # Announcement
 for o in Announcement.objects.all().select_related('announced_to', 'announced_from').order_by('announcement_id').iterator():
+    print "importing Announcement", o.pk
     try:
         message = Message.objects.get(id=o.announcement_id)
     except Message.DoesNotExist:
@@ -74,7 +74,7 @@
             message.bcc = l[len("bcc:"):].strip()
         elif l.lower().startswith("reply-to:"):
             message.reply_to = l[len("reply-to:"):].strip()
-    message.text = o.text
+    message.body = o.text
     message.save()
 
     message.related_groups.clear()
@@ -86,3 +86,77 @@
 
         message.related_groups.add(nomcom)
 
+
+# precompute scheduled_by's to speed up the loop a bit
+scheduled_by_mapping = {}
+for by in ScheduledAnnouncement.objects.all().values_list("scheduled_by", flat=True).distinct():
+    logins = IESGLogin.objects.filter(login_name=by)
+    if logins:
+        l = logins[0]
+        person = l.person
+        if not person:
+            person = PersonOrOrgInfo.objects.get(first_name=l.first_name, last_name=l.last_name)
+        found = old_person_to_person(person)
+    else:
+        found = system
+
+    print "mapping", by, "to", found
+    scheduled_by_mapping[by] = found
+
+# ScheduledAnnouncement
+for o in ScheduledAnnouncement.objects.all().order_by('id').iterator():
+    print "importing ScheduledAnnouncement", o.pk
+    try:
+        q = SendQueue.objects.get(id=o.id)
+    except SendQueue.DoesNotExist:
+        q = SendQueue(id=o.id)
+        # make sure there's no id overlap with ordinary already-imported announcements
+        q.message = Message(id=o.id + 4000)
+
+    time = datetime.datetime.combine(o.scheduled_date,
+                                     datetime.time(*(int(x) for x in o.scheduled_time.split(":"))))
+    by = scheduled_by_mapping[o.scheduled_by]
+
+    q.message.time = time
+    q.message.by = by
+    
+    q.message.subject = (o.subject or "").strip()
+    q.message.to = (o.to_val or "").strip()
+    q.message.frm = (o.from_val or "").strip()
+    q.message.cc = (o.cc_val or "").strip()
+    q.message.bcc = (o.bcc_val or "").strip()
+    q.message.reply_to = (o.replyto or "").strip()
+    q.message.body = o.body or ""
+    q.message.content_type = o.content_type or ""
+    q.message.save()
+
+    q.time = time
+    q.by = by
+
+    d = None
+    if o.to_be_sent_date:
+        try:
+            t = datetime.time(*(int(x) for x in o.to_be_sent_time.split(":")))
+        except ValueError:
+            t = datetime.time(0, 0, 0)
+        d = datetime.datetime.combine(o.to_be_sent_date, t)
+    
+    q.send_at = d
+
+    d = None
+    if o.actual_sent_date:
+        try:
+            t = datetime.time(*(int(x) for x in o.scheduled_time.split(":")))
+        except ValueError:
+            t = datetime.time(0, 0, 0)
+
+        d = datetime.datetime.combine(o.actual_sent_date, t)
+
+    q.sent_at = d
+
+    n = (o.note or "").strip()
+    if n.startswith("<br>"):
+        n = n[len("<br>"):]
+    q.note = n
+    
+    q.save()