This is a series of 50 migrations that changes the Document and DocAlias

primary keys from character strings to integers, and makes corresponding code
changes.

This was prompted by database limitations discovered when trying to make
DocAlias use a m2m document field; with 255 long strings as primary keys for
Document and DocAlias this violated the MySQL database limitations.

Changing the primary keys to integers should also improve efficiency.  

Due to the data migrations which create the new integer primary keys and adds
corresponding integer foreign keys matching the previous string foreign keys
in all tables having foreign keys to Document and DocAlias, some of these
migrations take a long time.  The total set of migrations are expected to have
a runtime on the order of 2 hours.
 - Legacy-Id: 16237

Author: levkowetz

ietf/community/migrations/0003_add_communitylist_docs2_m2m.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-21 14:23
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import ietf.utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('community', '0002_auto_20180220_1052'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CommunityListDocs',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('communitylist', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='community.CommunityList')),
+                ('document', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='doc.Document', to_field=b'id')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='communitylist',
+            name='added_docs2',
+            field=models.ManyToManyField(related_name='communitylists', through='community.CommunityListDocs', to='doc.Document'),
+        ),
+        migrations.CreateModel(
+            name='SearchRuleDocs',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('document', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='doc.Document', to_field=b'id')),
+                ('searchrule', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='community.SearchRule')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='searchrule',
+            name='name_contains_index2',
+            field=models.ManyToManyField(related_name='searchrules', through='community.SearchRuleDocs', to='doc.Document'),
+        ),
+    ]

ietf/community/migrations/0004_set_document_m2m_keys.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-21 14:27
+from __future__ import unicode_literals
+
+import sys
+
+from tqdm import tqdm
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+
+    Document                    = apps.get_model('doc','Document')
+    CommunityList               = apps.get_model('community', 'CommunityList')
+    CommunityListDocs           = apps.get_model('community', 'CommunityListDocs')
+    SearchRule                  = apps.get_model('community', 'SearchRule')
+    SearchRuleDocs              = apps.get_model('community', 'SearchRuleDocs')
+
+    # Document id fixup ------------------------------------------------------------
+
+    objs = Document.objects.in_bulk()
+    nameid = { o.name: o.id for id, o in objs.iteritems() }
+
+    sys.stderr.write('\n')
+
+    sys.stderr.write(' %s.%s:\n' % (CommunityList.__name__, 'added_docs'))
+    count = 0
+    for l in tqdm(CommunityList.objects.all()):
+        for d in l.added_docs.all():
+            count += 1
+            CommunityListDocs.objects.get_or_create(communitylist=l, document_id=nameid[d.name])
+    sys.stderr.write(' %s CommunityListDocs objects created\n' % (count, ))
+
+    sys.stderr.write(' %s.%s:\n' % (SearchRule.__name__, 'name_contains_index'))
+    count = 0
+    for r in tqdm(SearchRule.objects.all()):
+        for d in r.name_contains_index.all():
+            count += 1
+            SearchRuleDocs.objects.get_or_create(searchrule=r, document_id=nameid[d.name])
+    sys.stderr.write(' %s SearchRuleDocs objects created\n' % (count, ))
+
+def reverse(apps, schema_editor):
+    pass
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('community', '0003_add_communitylist_docs2_m2m'),
+        ('doc', '0014_set_document_docalias_id'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]

ietf/community/migrations/0005_1_del_docs_m2m_table.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-22 08:15
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('community', '0004_set_document_m2m_keys'),
+    ]
+
+    # The implementation of AlterField in Django 1.11 applies
+    #   'ALTER TABLE <table> MODIFY <field> ...;' in order to fix foregn keys
+    #   to the altered field, but as it seems does _not_ fix up m2m
+    #   intermediary tables in an equivalent manner, so here we remove and
+    #   then recreate the m2m tables so they will have the appropriate field
+    #   types.
+
+    operations = [
+        # Remove fields
+        migrations.RemoveField(
+            model_name='communitylist',
+            name='added_docs',
+        ),
+        migrations.RemoveField(
+            model_name='searchrule',
+            name='name_contains_index',
+        ),
+    ]

ietf/community/migrations/0005_2_add_docs_m2m_table.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-22 08:15
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('community', '0005_1_del_docs_m2m_table'),
+    ]
+
+    # The implementation of AlterField in Django 1.11 applies
+    #   'ALTER TABLE <table> MODIFY <field> ...;' in order to fix foregn keys
+    #   to the altered field, but as it seems does _not_ fix up m2m
+    #   intermediary tables in an equivalent manner, so here we remove and
+    #   then recreate the m2m tables so they will have the appropriate field
+    #   types.
+
+    operations = [
+        # Add fields back (will create the m2m tables with the right field types)
+        migrations.AddField(
+            model_name='communitylist',
+            name='added_docs',
+            field=models.ManyToManyField(to='doc.Document'),
+        ),
+        migrations.AddField(
+            model_name='searchrule',
+            name='name_contains_index',
+            field=models.ManyToManyField(to='doc.Document'),
+        ),
+    ]

ietf/community/migrations/0006_copy_docs_m2m_table.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-27 05:56
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+import sys, time
+
+from tqdm import tqdm
+
+
+def forward(apps, schema_editor):
+
+    CommunityList               = apps.get_model('community', 'CommunityList')
+    CommunityListDocs           = apps.get_model('community', 'CommunityListDocs')
+    SearchRule                  = apps.get_model('community', 'SearchRule')
+    SearchRuleDocs              = apps.get_model('community', 'SearchRuleDocs')
+
+    # Document id fixup ------------------------------------------------------------
+
+
+    sys.stderr.write('\n')
+
+    sys.stderr.write(' %s.%s:\n' % (CommunityList.__name__, 'added_docs'))
+    for l in tqdm(CommunityList.objects.all()):
+        l.added_docs.set([ d.document for d in CommunityListDocs.objects.filter(communitylist=l) ])
+
+    sys.stderr.write(' %s.%s:\n' % (SearchRule.__name__, 'name_contains_index'))
+    for r in tqdm(SearchRule.objects.all()):
+        r.name_contains_index.set([ d.document for d in SearchRuleDocs.objects.filter(searchrule=r) ])
+
+def reverse(apps, schema_editor):
+    pass
+
+def timestamp(apps, schema_editor):
+    sys.stderr.write('\n %s' % time.strftime('%Y-%m-%d %H:%M:%S'))
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('community', '0005_2_add_docs_m2m_table'),
+    ]
+
+    operations = [
+        #migrations.RunPython(forward, reverse),
+        # Alternative:
+        migrations.RunPython(timestamp, timestamp),
+        migrations.RunSQL(
+            "INSERT INTO community_communitylist_added_docs SELECT * FROM community_communitylistdocs;",
+            ""),
+        migrations.RunPython(timestamp, timestamp),
+        migrations.RunSQL(
+            "INSERT INTO community_searchrule_name_contains_index SELECT * FROM community_searchruledocs;",
+            ""),
+        migrations.RunPython(timestamp, timestamp),
+    ]

ietf/community/migrations/0007_remove_docs2_m2m.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-30 03:06
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('community', '0006_copy_docs_m2m_table'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='communitylistdocs',
+            name='communitylist',
+        ),
+        migrations.RemoveField(
+            model_name='communitylistdocs',
+            name='document',
+        ),
+        migrations.RemoveField(
+            model_name='searchruledocs',
+            name='document',
+        ),
+        migrations.RemoveField(
+            model_name='searchruledocs',
+            name='searchrule',
+        ),
+        migrations.RemoveField(
+            model_name='communitylist',
+            name='added_docs2',
+        ),
+        migrations.RemoveField(
+            model_name='searchrule',
+            name='name_contains_index2',
+        ),
+        migrations.DeleteModel(
+            name='CommunityListDocs',
+        ),
+        migrations.DeleteModel(
+            name='SearchRuleDocs',
+        ),
+    ]

ietf/community/tests.py

@@ -107,7 +107,7 @@ def test_manage_personal_list(self):
         self.assertEqual(r.status_code, 200)
 
         # add document
-        r = self.client.post(url, { "action": "add_documents", "documents": draft.pk })
+        r = self.client.post(url, { "action": "add_documents", "documents": draft.name })
         self.assertEqual(r.status_code, 302)
         clist = CommunityList.objects.get(user__username="plain")
         self.assertTrue(clist.added_docs.filter(pk=draft.pk))
@@ -118,7 +118,7 @@ def test_manage_personal_list(self):
         self.assertTrue(draft.name in unicontent(r))
 
         # remove document
-        r = self.client.post(url, { "action": "remove_document", "document": draft.pk })
+        r = self.client.post(url, { "action": "remove_document", "document": draft.name })
         self.assertEqual(r.status_code, 302)
         clist = CommunityList.objects.get(user__username="plain")
         self.assertTrue(not clist.added_docs.filter(pk=draft.pk))

ietf/community/utils.py

@@ -61,7 +61,7 @@ def augment_docs_with_tracking_info(docs, user):
     if user and user.is_authenticated:
         clist = CommunityList.objects.filter(user=user).first()
         if clist:
-            tracked.update(docs_tracked_by_community_list(clist).filter(pk__in=docs).values_list("pk", flat=True))
+            tracked.update(docs_tracked_by_community_list(clist).filter(pk__in=[ d.pk for d in docs ]).values_list("pk", flat=True))
 
     for d in docs:
         d.tracked_in_personal_community_list = d.pk in tracked

ietf/community/views.py

@@ -57,9 +57,9 @@ def manage_list(request, username=None, acronym=None, group_type=None):
         add_doc_form = AddDocumentsForm()
 
     if request.method == 'POST' and action == 'remove_document':
-        document_pk = request.POST.get('document')
-        if clist.pk is not None and document_pk:
-            document = get_object_or_404(clist.added_docs, pk=document_pk)
+        document_name = request.POST.get('document')
+        if clist.pk is not None and document_name:
+            document = get_object_or_404(clist.added_docs, name=document_name)
             clist.added_docs.remove(document)
 
             return HttpResponseRedirect("")
@@ -209,7 +209,7 @@ def feed(request, username=None, acronym=None, group_type=None):
     since = datetime.datetime.now() - datetime.timedelta(days=14)
 
     events = DocEvent.objects.filter(
-        doc__in=documents,
+        doc__id__in=documents,
         time__gte=since,
     ).distinct().order_by('-time', '-id').select_related("doc")
 

ietf/doc/fields.py

@@ -2,6 +2,7 @@
 
 from django.utils.html import escape
 from django import forms
+from django.db.models import Q
 from django.urls import reverse as urlreverse
 
 import debug                            # pyflakes:ignore
@@ -52,8 +53,10 @@ def prepare_value(self, value):
         if isinstance(value, (int, long)):
             value = str(value)
         if isinstance(value, basestring):
-            pks = self.parse_select2_value(value)
-            value = self.model.objects.filter(pk__in=pks)
+            items = self.parse_select2_value(value)
+            names = [ i for i in items if not i.isdigit() ]
+            ids   = [ i for i in items if i.isdigit() ]
+            value = self.model.objects.filter(Q(name__in=names)|Q(id__in=ids))
             filter_args = {}
             if self.model == DocAlias:
                 filter_args["document__type"] = self.doc_type
@@ -76,17 +79,17 @@ def prepare_value(self, value):
 
     def clean(self, value):
         value = super(SearchableDocumentsField, self).clean(value)
-        pks = self.parse_select2_value(value)
+        names = self.parse_select2_value(value)
 
-        objs = self.model.objects.filter(pk__in=pks)
+        objs = self.model.objects.filter(name__in=names)
 
-        found_pks = [str(o.pk) for o in objs]
-        failed_pks = [x for x in pks if x not in found_pks]
-        if failed_pks:
-            raise forms.ValidationError(u"Could not recognize the following documents: {pks}. You can only input documents already registered in the Datatracker.".format(pks=", ".join(failed_pks)))
+        found_names = [str(o.name) for o in objs]
+        failed_names = [x for x in names if x not in found_names]
+        if failed_names:
+            raise forms.ValidationError(u"Could not recognize the following documents: {names}. You can only input documents already registered in the Datatracker.".format(names=", ".join(failed_names)))
 
         if self.max_entries != None and len(objs) > self.max_entries:
-            raise forms.ValidationError(u"You can select at most %s entries only." % self.max_entries)
+            raise forms.ValidationError(u"You can select at most %s entries." % self.max_entries)
 
         return objs