datatracker/ietf/mailinglists/management/commands/import_mailman_listinfo.py at git-conversion · https-github-com-bit/datatracker · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
# Copyright The IETF Trust 2016-2019, All Rights Reserved

import sys
import time
from textwrap import dedent

import debug                            # pyflakes:ignore

from django.conf import settings
from django.core.management.base import BaseCommand
from django.core.exceptions import MultipleObjectsReturned

sys.path.append(settings.MAILMAN_LIB_DIR)

have_mailman = False
try:
    from Mailman import Utils
    from Mailman import MailList
    from Mailman import MemberAdaptor
    have_mailman = True
except ImportError:
    pass

from ietf.mailinglists.models import List, Subscribed
from ietf.utils.log import log
from ietf.utils.text import decode

mark = time.time()

def import_mailman_listinfo(verbosity=0):
    def note(msg):
        if verbosity > 2:
            sys.stdout.write(msg)
            sys.stdout.write('\n')
    def log_time(msg):
        global mark
        if verbosity > 1:
            t = time.time()
            log(msg+' (%.1fs)'% (t-mark))
            mark = t

    if not have_mailman:
        note("Could not import mailman modules -- skipping import of mailman list info")
        return

    log("Starting import of list info from Mailman")
    names = list(Utils.list_names())
    names.sort()
    log_time("Fetched list of mailman list names")
    addr_max_length = Subscribed._meta.get_field('email').max_length

    subscribed = { l.name: set(l.subscribed_set.values_list('email', flat=True)) for l in List.objects.all().prefetch_related('subscribed_set') }
    log_time("Computed dictionary of list members")

    for name in names:
        mlist = MailList.MailList(name, lock=False)
        note("List: %s" % mlist.internal_name())
        log_time("Fetched Mailman list object for %s" % name)

        lists = List.objects.filter(name=mlist.real_name)
        if lists.count() > 1:
            # Arbitrary choice; we'll update the remaining item next
            for item in lists[1:]:
                item.delete()
        mmlist, created = List.objects.get_or_create(name=mlist.real_name)
        dirty = False
        desc = decode(mlist.description)[:256]
        if mmlist.description != desc:
            mmlist.description = desc
            dirty = True
        if mmlist.advertised != mlist.advertised:
            mmlist.advertised = mlist.advertised
            dirty = True
        if dirty:
            mmlist.save()
        log_time("  Updated database List object for %s" % name)
        # The following calls return lowercased addresses
        if mlist.advertised:
            members = mlist.getRegularMemberKeys() + mlist.getDigestMemberKeys()
            log_time("  Fetched list of list members")
            members = set([ m for m in members if mlist.getDeliveryStatus(m) == MemberAdaptor.ENABLED ])
            log_time("  Filtered list of list members")
            if not mlist.real_name in subscribed:
                log("Note: didn't find '%s' in the dictionary of subscriptions" % mlist.real_name)
                continue
            known = subscribed[mlist.real_name]
            log_time("  Fetched known list members from database")
            to_remove = known - members
            to_add = members - known
            for addr in to_remove:
                    note("  Removing subscription: %s" % (addr))
                    old = Subscribed.objects.get(email=addr)
                    log_time("    Fetched subscribed object")
                    old.lists.remove(mmlist)
                    log_time("    Removed %s from %s" % (mmlist, old))
                    if old.lists.count() == 0:
                        note("    Removing address with no subscriptions: %s" % (addr))
                        old.delete()
                        log_time("      Removed %s" % old)
            log_time("  Removed addresses no longer subscribed")
            if to_remove:
                log("  Removed %s addresses from %s" % (len(to_remove), name))
            for addr in to_add:
                if len(addr) > addr_max_length:
                    sys.stderr.write("    **  Email address subscribed to '%s' too long for table: <%s>\n" % (name, addr))
                    continue
                note("  Adding subscription: %s" % (addr))
                try:
                    new, created = Subscribed.objects.get_or_create(email=addr)
                except MultipleObjectsReturned as e:
                    sys.stderr.write("    **  Error handling %s in %s: %s\n" % (addr, name, e))
                    continue
                new.lists.add(mmlist)
            log_time("  Added new addresses")
            if to_add:
                log("  Added %s addresses to %s" % (len(to_add), name))
    log("Completed import of list info from Mailman")

class Command(BaseCommand):
    """
    Import list information from Mailman.

    Import announced list names, descriptions, and subscribers, by calling the
    appropriate Mailman functions and adding entries to the database.

    Run this from cron regularly, with sufficient permissions to access the
    mailman database files.

    """

    help = dedent(__doc__).strip()

    #option_list = BaseCommand.option_list + (       )


    def handle(self, *filenames, **options):
        """

        * Import announced lists, with appropriate meta-information.

        * For each list, import the members.

        """

        verbosity = int(options.get('verbosity'))

        import_mailman_listinfo(verbosity)