Speed up the community refactor migration by building the index

of the group name search rules in Python
 - Legacy-Id: 11188

Author: OleLaursen

ietf/community/migrations/0004_cleanup_data.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-from tqdm import tqdm
+import re
 from django.db import migrations, models
 
 def port_rules_to_typed_system(apps, schema_editor):
@@ -22,11 +22,7 @@ def try_to_uniquify_person(rule, person_qs):
         return person_qs
 
 
-    print("")
-    print("   * Port Rules to Typed System")
-    print("     Expect an initial rule count of around 550 here, as of May 2016:")
-    print("     rule count: %s" % SearchRule.objects.all().count())
-    for rule in tqdm(SearchRule.objects.all()):
+    for rule in SearchRule.objects.all():
         handled = False
 
         if rule.rule_type in ['wg_asociated', 'area_asociated', 'wg_asociated_rfc', 'area_asociated_rfc']:
@@ -117,14 +113,10 @@ def try_to_uniquify_person(rule, person_qs):
         else:
             rule.delete()
             #print "NOT HANDLED", rule.pk, rule.rule_type, rule.value
-    print("     rule count: %s" % SearchRule.objects.all().count())
 
 def delete_extra_person_rules(apps, schema_editor):
     SearchRule = apps.get_model("community", "SearchRule")
-    print("")
-    print("   * Delete Extra Person Rules")
     SearchRule.objects.exclude(person=None).filter(value="").delete()
-    print("     rule count: %s" % SearchRule.objects.all().count())
 
 RENAMED_RULES = [
     ('wg_asociated', 'group'),
@@ -150,44 +142,33 @@ def rename_rule_type_forwards(apps, schema_editor):
 
     renamings = dict(RENAMED_RULES)
 
-    print("")
-    print("   * Rename Rule Type Forwards")
-    for r in tqdm(SearchRule.objects.all()):
+    for r in SearchRule.objects.all():
         if r.rule_type in renamings:
             r.rule_type = renamings[r.rule_type]
             r.save()
-    print("     rule count: %s" % SearchRule.objects.all().count())
 
 def rename_rule_type_backwards(apps, schema_editor):
     SearchRule = apps.get_model("community", "SearchRule")
 
     renamings = dict((to, fro) for fro, to in RENAMED_RULES)
 
-    print("")
-    print("   * Rename Rule Type Backwards")
-    for r in tqdm(SearchRule.objects.all()):
+    for r in SearchRule.objects.all():
         if r.rule_type in renamings:
             r.rule_type = renamings[r.rule_type]
             r.save()
-    print("     rule count: %s" % SearchRule.objects.all().count())
 
 def get_rid_of_empty_lists(apps, schema_editor):
     CommunityList = apps.get_model("community", "CommunityList")
 
-    print("")
-    print("   * Get Rid of Empty Lists")
-    for cl in tqdm(CommunityList.objects.all()):
-        if not cl.added_docs.exists() and not cl.searchrule_set.exists() and not cl.emailsubscription_set.exists():
-            cl.delete()
+    for cl in CommunityList.objects.filter(added_docs=None, searchrule=None, emailsubscription=None):
+        cl.delete()
 
 def move_email_subscriptions_to_preregistered_email(apps, schema_editor):
     EmailSubscription = apps.get_model("community", "EmailSubscription")
     Email = apps.get_model("person", "Email")
     Person = apps.get_model("person", "Person")
 
-    print("")
-    print("   * Move Email Subscriptions to Preregistered Email")
-    for e in tqdm(EmailSubscription.objects.all()):
+    for e in EmailSubscription.objects.all():
         email_obj = None
         try:
             email_obj = Email.objects.get(address=e.email)
@@ -210,38 +191,40 @@ def move_email_subscriptions_to_preregistered_email(apps, schema_editor):
 def fill_in_notify_on(apps, schema_editor):
     EmailSubscription = apps.get_model("community", "EmailSubscription")
 
-#    print("")
-#    print("   * Fill In Notify On")
     EmailSubscription.objects.filter(significant=False).update(notify_on="all")
     EmailSubscription.objects.filter(significant=True).update(notify_on="significant")
 
 def add_group_community_lists(apps, schema_editor):
     Group = apps.get_model("group", "Group")
-    Document = apps.get_model("doc", "Document")
+    DocAlias = apps.get_model("doc", "DocAlias")
     State = apps.get_model("doc", "State")
     CommunityList = apps.get_model("community", "CommunityList")
     SearchRule = apps.get_model("community", "SearchRule")
 
     active_state = State.objects.get(slug="active", type="draft")
     rfc_state = State.objects.get(slug="rfc", type="draft")
 
-    print("")
-    print("   * Add Group Community Lists")
-    for g in tqdm(Group.objects.filter(type__in=("rg", "wg"))):
+    draft_aliases = DocAlias.objects.filter(name__startswith="draft")
+
+    for g in Group.objects.filter(type__in=("rg", "wg")):
         clist = CommunityList.objects.filter(group=g).first()
         if clist:
             SearchRule.objects.get_or_create(community_list=clist, rule_type="group", group=g, state=active_state)
             SearchRule.objects.get_or_create(community_list=clist, rule_type="group_rfc", group=g, state=rfc_state)
             r, _ = SearchRule.objects.get_or_create(community_list=clist, rule_type="name_contains", text=r"^draft-[^-]+-%s-" % g.acronym, state=active_state)
-            r.name_contains_index = Document.objects.filter(docalias__name__regex=r.text)
+
+            # do the matching manually in Python to speed up the operation
+            #r.name_contains_index = Document.objects.filter(docalias__name__regex=r.text)
+            name_re = re.compile(r.text)
+            r.name_contains_index = [ a.document_id for a in draft_aliases if name_re.match(a.name) ]
 
         else:
             clist = CommunityList.objects.create(group=g)
             SearchRule.objects.create(community_list=clist, rule_type="group", group=g, state=active_state)
             SearchRule.objects.create(community_list=clist, rule_type="group_rfc", group=g, state=rfc_state)
             r = SearchRule.objects.create(community_list=clist, rule_type="name_contains", text=r"^draft-[^-]+-%s-" % g.acronym, state=active_state)
-            r.name_contains_index = Document.objects.filter(docalias__name__regex=r.text)
-    print("     rule count: %s" % SearchRule.objects.all().count())
+            name_re = re.compile(r.text)
+            r.name_contains_index = [ a.document_id for a in draft_aliases if name_re.match(a.name) ]
 
 def noop(apps, schema_editor):
     pass