Added models for mailman lists and subscribed and whitelisted addresses, and an import management command suited to be run as a periodic cron-job.

 - Legacy-Id: 11373

Author: levkowetz

ietf/mailinglists/admin.py

@@ -0,0 +1,23 @@
+# Copyright The IETF Trust 2016, All Rights Reserved
+
+from django.contrib import admin
+
+from ietf.mailinglists.models import List, Subscribed, Whitelisted
+
+
+class ListAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name', 'description', 'advertised')
+    search_fields = ('name',)
+admin.site.register(List, ListAdmin)
+
+
+class SubscribedAdmin(admin.ModelAdmin):
+    list_display = ('id', 'time', 'address')
+    raw_id_fields = ('lists',)
+    search_fields = ('address',)
+admin.site.register(Subscribed, SubscribedAdmin)
+
+
+class WhitelistedAdmin(admin.ModelAdmin):
+    list_display = ('id', 'time', 'address', 'by')
+admin.site.register(Whitelisted, WhitelistedAdmin)

ietf/mailinglists/management/__init__.py

@@ -0,0 +1,61 @@
+# Copyright The IETF Trust 2016, All Rights Reserved
+
+import sys
+from textwrap import dedent
+
+import debug                            # pyflakes:ignore
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from ietf.mailinglists.models import List, Subscribed
+
+class Command(BaseCommand):
+    """
+    Import list information from Mailman.
+
+    Import announced list names, descriptions, and subscribers, by calling the
+    appropriate Mailman functions and adding entries to the database.
+
+    Run this from cron regularly, with sufficient permissions to access the
+    mailman database files.
+
+    """
+
+    help = dedent(__doc__).strip()
+            
+    #option_list = BaseCommand.option_list + (       )
+
+    def note(self, msg):    
+        if self.verbosity > 1:
+            self.stdout.write(msg)
+
+    def handle(self, *filenames, **options):
+        """
+
+        * Import announced lists, with appropriate meta-information.
+
+        * For each list, import the members.
+
+        """
+
+        self.verbosity = int(options.get('verbosity'))
+
+        sys.path.append(settings.MAILMAN_LIB_DIR)
+
+        from Mailman import Utils
+        from Mailman import MailList
+
+        for name in Utils.list_names():
+            mlist = MailList.MailList(name, lock=False)
+            self.note("List: %s" % mlist.internal_name())
+            if mlist.advertised:
+                list, created = List.objects.get_or_create(name=mlist.real_name, description=mlist.description, advertised=mlist.advertised)
+                # The following calls return lowercased addresses
+                members = mlist.getRegularMemberKeys() + mlist.getDigestMemberKeys()
+                known = [ s.address for s in Subscribed.objects.filter(lists__name=name) ]
+                for addr in members:
+                    if not addr in known:
+                        self.note("  Adding subscribed: %s" % (addr))
+                        new, created = Subscribed.objects.get_or_create(address=addr)
+                        new.lists.add(list)

ietf/mailinglists/management/commands/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+from django.db import migrations
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+    ]

ietf/mailinglists/management/commands/import_mailman_listinfo.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+import django.core.validators
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('person', '0014_auto_20160613_0751'),
+        ('mailinglists', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='List',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('name', models.CharField(max_length=32)),
+                ('description', models.CharField(max_length=256)),
+                ('advertised', models.BooleanField(default=True)),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name='Subscribed',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('time', models.DateTimeField(auto_now_add=True)),
+                ('address', models.CharField(max_length=64, validators=[django.core.validators.EmailValidator()])),
+                ('lists', models.ManyToManyField(to='mailinglists.List')),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name='Whitelisted',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('time', models.DateTimeField(auto_now_add=True)),
+                ('address', models.CharField(max_length=64, validators=[django.core.validators.EmailValidator()])),
+                ('by', models.ForeignKey(to='person.Person')),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+    ]

ietf/mailinglists/migrations/0001_initial.py

@@ -0,0 +1,58 @@
+# Copyright The IETF Trust 2016, All Rights Reserved
+# Autogenerated by the makeresources management command 2016-06-12 12:29 PDT
+from tastypie.resources import ModelResource
+from tastypie.fields import ToManyField                 # pyflakes:ignore
+from tastypie.constants import ALL, ALL_WITH_RELATIONS  # pyflakes:ignore
+from tastypie.cache import SimpleCache
+
+from ietf import api
+from ietf.api import ToOneField                         # pyflakes:ignore
+
+from ietf.mailinglists.models import Whitelisted, List, Subscribed
+
+
+from ietf.person.resources import PersonResource
+class WhitelistedResource(ModelResource):
+    by               = ToOneField(PersonResource, 'by')
+    class Meta:
+        queryset = Whitelisted.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'whitelisted'
+        filtering = { 
+            "id": ALL,
+            "time": ALL,
+            "address": ALL,
+            "by": ALL_WITH_RELATIONS,
+        }
+api.mailinglists.register(WhitelistedResource())
+
+class ListResource(ModelResource):
+    class Meta:
+        queryset = List.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'list'
+        filtering = { 
+            "id": ALL,
+            "name": ALL,
+            "description": ALL,
+            "advertised": ALL,
+        }
+api.mailinglists.register(ListResource())
+
+class SubscribedResource(ModelResource):
+    lists            = ToManyField(ListResource, 'lists', null=True)
+    class Meta:
+        queryset = Subscribed.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'subscribed'
+        filtering = { 
+            "id": ALL,
+            "time": ALL,
+            "address": ALL,
+            "lists": ALL_WITH_RELATIONS,
+        }
+api.mailinglists.register(SubscribedResource())
+

ietf/mailinglists/migrations/0002_list_subscribed_whitelisted.py

@@ -1,3 +1,5 @@
+# Copyright The IETF Trust 2016, All Rights Reserved
+
 from django.core.urlresolvers import reverse as urlreverse
 
 from pyquery import PyQuery

ietf/mailinglists/migrations/__init__.py

@@ -650,6 +650,14 @@ def skip_unreadable_post(record):
     }
 }
 
+MAILMAN_LIB_DIR = '/usr/lib/mailman'
+
+# This is the number of seconds required between subscribing to an ietf
+# mailing list and datatracker account creation being accepted
+LIST_ACCOUNT_DELAY = 60*60*25           # 25 hours
+ACCOUNT_REQUEST_EMAIL = 'account-request@ietf.org'
+
+
 # Put the production SECRET_KEY in settings_local.py, and also any other
 # sensitive or site-specific changes.  DO NOT commit settings_local.py to svn.
 from settings_local import *            # pyflakes:ignore