feat: isolate py27 code interfacing with mailman from py39 code interfacing with django. (ietf-tools#4140)

* feat: isolate py27 code interfacing with mailman from py39 code interfacing with django.

* fix: improve memory footprint and remove unneeded import.

* fix: make new bin command executable.

Author: rjsparks

bin/mm_hourly

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Hourly datatracker jobs, run as mailman
+# Hourly datatracker jobs, ***run as mailman***
 # 
 # This script is expected to be triggered by cron from
 # $DTDIR/etc/cron.d/datatracker which should be symlinked from
@@ -13,14 +13,12 @@ export PYTHONIOENCODING=utf-8
 program=${0##*/}
 trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
 
-# Note that we're using the last 2.7 release here, not the current release
-DTDIR=/a/www/ietf-datatracker/py27
+DTDIR=/a/www/ietf-datatracker/web
 cd $DTDIR/
 
 # Set up the virtual environment
 source $DTDIR/env/bin/activate
 
 logger -p user.info -t cron "Running $DTDIR/bin/mm_hourly"
 
-## XXX commented out pending rewrite -- mailman 2 python interface is not available under Python 3
 $DTDIR/ietf/manage.py import_mailman_listinfo

ietf/bin/mailman_listinfo.py

@@ -0,0 +1,50 @@
+#!/usr/bin/python2.7
+# Copyright The IETF Trust 2022, All Rights Reserved
+# Note the shebang. This specifically targets deployment on IETFA and intends to use its system python2.7.
+
+# This is an adaptor to pull information out of Mailman2 using its python libraries (which are only available for python2).
+# It is NOT django code, and does not have access to django.conf.settings.
+
+import json
+import sys
+
+from collections import defaultdict
+
+def main():
+
+    sys.path.append('/usr/lib/mailman')
+
+    have_mailman = False
+    try:
+        from Mailman import Utils
+        from Mailman import MailList
+        from Mailman import MemberAdaptor
+        have_mailman = True
+    except ImportError:
+        pass
+
+
+    if not have_mailman:
+        sys.stderr.write("Could not import mailman modules -- skipping import of mailman list info")
+        sys.exit()
+
+    names = list(Utils.list_names())
+
+    # need to emit dict of names, each name has an mlist, and each mlist has description, advertised, and members (calculated as below)
+    result = defaultdict(dict)
+    for name in names:
+        mlist = MailList.MailList(name, lock=False)
+        result[name] = dict()
+        result[name]['internal_name'] = mlist.internal_name()
+        result[name]['real_name'] = mlist.real_name
+        result[name]['description'] = mlist.description # Not attempting to change encoding
+        result[name]['advertised'] = mlist.advertised
+        result[name]['members'] = list()
+        if mlist.advertised:
+            members = mlist.getRegularMemberKeys() + mlist.getDigestMemberKeys()
+            members = set([ m for m in members if mlist.getDeliveryStatus(m) == MemberAdaptor.ENABLED ])
+            result[name]['members'] = list(members)
+    json.dump(result, sys.stdout)
+
+if __name__ == "__main__":
+    main()

ietf/mailinglists/management/commands/import_mailman_listinfo.py

@@ -1,29 +1,22 @@
 # Copyright The IETF Trust 2016-2019, All Rights Reserved
 
+import json
 import sys
+import subprocess
 import time
 from textwrap import dedent
 
 import debug                            # pyflakes:ignore
 
+from pathlib import Path
+
 from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.core.exceptions import MultipleObjectsReturned
 
-sys.path.append(settings.MAILMAN_LIB_DIR)
-
-have_mailman = False
-try:
-    from Mailman import Utils
-    from Mailman import MailList
-    from Mailman import MemberAdaptor
-    have_mailman = True
-except ImportError:
-    pass
 
 from ietf.mailinglists.models import List, Subscribed
 from ietf.utils.log import log
-from ietf.utils.text import decode
 
 mark = time.time()
 
@@ -39,51 +32,50 @@ def log_time(msg):
             log(msg+' (%.1fs)'% (t-mark))
             mark = t
 
-    if not have_mailman:
-        note("Could not import mailman modules -- skipping import of mailman list info")
+    cmd = str(Path(settings.BASE_DIR) / "bin" / "mailman_listinfo.py")
+    result = subprocess.run([cmd], capture_output=True)
+    if result.stderr:
+        log("Error exporting information from mailmain")
+        log(result.stderr)
         return
+    mailman_export = json.loads(result.stdout)
 
     log("Starting import of list info from Mailman")
-    names = list(Utils.list_names())
-    names.sort()
+    names = sorted(mailman_export.keys())
     log_time("Fetched list of mailman list names")
     addr_max_length = Subscribed._meta.get_field('email').max_length
 
     subscribed = { l.name: set(l.subscribed_set.values_list('email', flat=True)) for l in List.objects.all().prefetch_related('subscribed_set') }
     log_time("Computed dictionary of list members")
 
     for name in names:
-        mlist = MailList.MailList(name, lock=False)
-        note("List: %s" % mlist.internal_name())
+        note("List: %s" % mailman_export[name]['internal_name'])
         log_time("Fetched Mailman list object for %s" % name)
 
-        lists = List.objects.filter(name=mlist.real_name)
+        lists = List.objects.filter(name=mailman_export[name]['real_name'])
         if lists.count() > 1:
             # Arbitrary choice; we'll update the remaining item next
             for item in lists[1:]:
                 item.delete()
-        mmlist, created = List.objects.get_or_create(name=mlist.real_name)
+        mmlist, created = List.objects.get_or_create(name=mailman_export[name]['real_name'])
         dirty = False
-        desc = decode(mlist.description)[:256]
+        desc = mailman_export[name]['description'][:256]
         if mmlist.description != desc:
             mmlist.description = desc
             dirty = True
-        if mmlist.advertised != mlist.advertised:
-            mmlist.advertised = mlist.advertised
+        if mmlist.advertised != mailman_export[name]['advertised']:
+            mmlist.advertised = mailman_export[name]['advertised']
             dirty = True
         if dirty:
             mmlist.save()
         log_time("  Updated database List object for %s" % name)
         # The following calls return lowercased addresses
-        if mlist.advertised:
-            members = mlist.getRegularMemberKeys() + mlist.getDigestMemberKeys()
-            log_time("  Fetched list of list members")
-            members = set([ m for m in members if mlist.getDeliveryStatus(m) == MemberAdaptor.ENABLED ])
-            log_time("  Filtered list of list members")
-            if not mlist.real_name in subscribed:
-                log("Note: didn't find '%s' in the dictionary of subscriptions" % mlist.real_name)
+        if mailman_export[name]['advertised']:
+            members = set(mailman_export[name]['members'])
+            if not mailman_export[name]['real_name'] in subscribed:
+                log("Note: didn't find '%s' in the dictionary of subscriptions" % mailman_export[name]['real_name'])
                 continue
-            known = subscribed[mlist.real_name]
+            known = subscribed[mailman_export[name]['real_name']]
             log_time("  Fetched known list members from database")            
             to_remove = known - members
             to_add = members - known