Merge author stats branch into new branch from trunk

 - Legacy-Id: 13159

Author: OleLaursen

ietf/api/__init__.py

@@ -212,9 +212,9 @@ def dehydrate(self, bundle, for_list=True):
         if not foreign_obj:
             if not self.null:
                 if callable(self.attribute):
-                    raise ApiFieldError("The related resource for resource %s could not be found." % (previous_obj))
+                    raise ApiFieldError(u"The related resource for resource %s could not be found." % (previous_obj))
                 else:
-                    raise ApiFieldError("The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attr))
+                    raise ApiFieldError(u"The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attr))
             return None
 
         fk_resource = self.get_related_resource(foreign_obj)

ietf/bin/find-submission-confirmation-email-in-postfix-log

@@ -66,7 +66,7 @@ if "<" in from_email:
 
 submission = Submission.objects.filter(name=draft).latest('submission_date')
 document = Document.objects.get(name=draft)
-emails = [ author.address for author in document.authors.all() ]
+emails = [ author.email.address for author in document.documentauthor_set.all() if author.email ]
 
 timestrings = []
 for file in [ Path(settings.INTERNET_DRAFT_PATH) / ("%s-%s.txt"%(draft, submission.rev)),

ietf/bin/generate-draft-aliases

@@ -65,7 +65,7 @@ def get_draft_authors_emails(draft):
     " Get list of authors for the given draft."
 
     # This feels 'correct'; however, it creates fairly large delta
-    return [email.email_address() for email in draft.authors.all()]
+    return [author.email.email_address() for author in draft.documentauthor_set.all() if author.email.email_address()]
 
     # This gives fairly small delta compared to current state,
     # however, it seems to be wrong (doesn't check for emails being

ietf/community/migrations/0004_cleanup_data.py

@@ -57,7 +57,7 @@ def try_to_uniquify_person(rule, person_qs):
 
 
         elif rule.rule_type in ["author", "author_rfc"]:
-            found_persons = list(try_to_uniquify_person(rule, Person.objects.filter(email__documentauthor__id__gte=1).filter(name__icontains=rule.value).distinct()))
+            found_persons = list(try_to_uniquify_person(rule, Person.objects.filter(documentauthor__id__gte=1).filter(name__icontains=rule.value).distinct()))
 
             if found_persons:
                 rule.person = found_persons[0]

ietf/community/tests.py

@@ -33,7 +33,7 @@ def test_rule_matching(self):
 
         rule_state_iesg = SearchRule.objects.create(rule_type="state_iesg", state=State.objects.get(type="draft-iesg", slug="lc"), community_list=clist)
 
-        rule_author = SearchRule.objects.create(rule_type="author", state=State.objects.get(type="draft", slug="active"), person=Person.objects.filter(email__documentauthor__document=draft).first(), community_list=clist)
+        rule_author = SearchRule.objects.create(rule_type="author", state=State.objects.get(type="draft", slug="active"), person=Person.objects.filter(documentauthor__document=draft).first(), community_list=clist)
 
         rule_ad = SearchRule.objects.create(rule_type="ad", state=State.objects.get(type="draft", slug="active"), person=draft.ad, community_list=clist)
 
@@ -115,7 +115,7 @@ def test_manage_personal_list(self):
         r = self.client.post(url, {
             "action": "add_rule",
             "rule_type": "author_rfc",
-            "author_rfc-person": Person.objects.filter(email__documentauthor__document=draft).first().pk,
+            "author_rfc-person": Person.objects.filter(documentauthor__document=draft).first().pk,
             "author_rfc-state": State.objects.get(type="draft", slug="rfc").pk,
         })
         self.assertEqual(r.status_code, 302)

ietf/community/utils.py

@@ -90,7 +90,7 @@ def docs_matching_community_list_rule(rule):
     elif rule.rule_type.startswith("state_"):
         return docs.filter(states=rule.state)
     elif rule.rule_type in ["author", "author_rfc"]:
-        return docs.filter(states=rule.state, documentauthor__author__person=rule.person)
+        return docs.filter(states=rule.state, documentauthor__person=rule.person)
     elif rule.rule_type == "ad":
         return docs.filter(states=rule.state, ad=rule.person)
     elif rule.rule_type == "shepherd":
@@ -123,7 +123,7 @@ def community_list_rules_matching_doc(doc):
     rules |= SearchRule.objects.filter(
         rule_type__in=["author", "author_rfc"],
         state__in=states,
-        person__in=list(Person.objects.filter(email__documentauthor__document=doc)),
+        person__in=list(Person.objects.filter(documentauthor__document=doc)),
     )
 
     if doc.ad_id:

ietf/doc/admin.py

@@ -26,7 +26,7 @@ class DocAliasInline(admin.TabularInline):
 
 class DocAuthorInline(admin.TabularInline):
     model = DocumentAuthor
-    raw_id_fields = ['author', ]    
+    raw_id_fields = ['person', 'email']
     extra = 1
 
 class RelatedDocumentInline(admin.TabularInline):
@@ -100,7 +100,7 @@ class DocumentAdmin(admin.ModelAdmin):
     list_display = ['name', 'rev', 'group', 'pages', 'intended_std_level', 'author_list', 'time']
     search_fields = ['name']
     list_filter = ['type']
-    raw_id_fields = ['authors', 'group', 'shepherd', 'ad']
+    raw_id_fields = ['group', 'shepherd', 'ad']
     inlines = [DocAliasInline, DocAuthorInline, RelatedDocumentInline, ]
     form = DocumentForm
 
@@ -123,7 +123,7 @@ class DocHistoryAdmin(admin.ModelAdmin):
     list_display = ['doc', 'rev', 'state', 'group', 'pages', 'intended_std_level', 'author_list', 'time']
     search_fields = ['doc__name']
     ordering = ['time', 'doc', 'rev']
-    raw_id_fields = ['doc', 'authors', 'group', 'shepherd', 'ad']
+    raw_id_fields = ['doc', 'group', 'shepherd', 'ad']
 
     def state(self, instance):
         return instance.get_state()
@@ -175,7 +175,8 @@ class BallotPositionDocEventAdmin(DocEventAdmin):
 admin.site.register(BallotPositionDocEvent, BallotPositionDocEventAdmin)
 
 class DocumentAuthorAdmin(admin.ModelAdmin):
-    list_display = ['id', 'document', 'author', 'order']
-    search_fields = [ 'document__name', 'author__address', ]
+    list_display = ['id', 'document', 'person', 'email', 'affiliation', 'country', 'order']
+    search_fields = ['document__docalias__name', 'person__name', 'email__address', 'affiliation', 'country']
+    raw_id_fields = ["document", "person", "email"]
 admin.site.register(DocumentAuthor, DocumentAuthorAdmin)
 

ietf/doc/factories.py

@@ -46,8 +46,8 @@ def states(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument
     def authors(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument
         if create and extracted:
             order = 0
-            for email in extracted:
-                DocumentAuthor.objects.create(document=obj, author=email, order=order)
+            for person in extracted:
+                DocumentAuthor.objects.create(document=obj, person=person, order=order)
                 order += 1
 
     @classmethod

ietf/doc/migrations/0026_author_revamp_and_extra_attributes.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name', '0019_formallanguagename'),
+        ('doc', '0025_auto_20170307_0146'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='dochistory',
+            name='words',
+            field=models.IntegerField(null=True, blank=True),
+        ),
+        migrations.AddField(
+            model_name='document',
+            name='words',
+            field=models.IntegerField(null=True, blank=True),
+        ),
+        migrations.AddField(
+            model_name='dochistory',
+            name='formal_languages',
+            field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True),
+        ),
+        migrations.AddField(
+            model_name='document',
+            name='formal_languages',
+            field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True),
+        ),
+        migrations.RemoveField(
+            model_name='dochistory',
+            name='authors',
+        ),
+        migrations.RemoveField(
+            model_name='document',
+            name='authors',
+        ),
+        migrations.AddField(
+            model_name='dochistoryauthor',
+            name='affiliation',
+            field=models.CharField(help_text=b'Organization/company used by author for submission', max_length=100, blank=True),
+        ),
+        migrations.AddField(
+            model_name='dochistoryauthor',
+            name='country',
+            field=models.CharField(blank=True, help_text=b'Country used by author for submission', max_length=255),
+        ),
+        migrations.RenameField(
+            model_name='dochistoryauthor',
+            old_name='author',
+            new_name='email',
+        ),
+        migrations.AlterField(
+            model_name='dochistoryauthor',
+            name='email',
+            field=models.ForeignKey(blank=True, to='person.Email', help_text=b'Email address used by author for submission', null=True),
+        ),
+        migrations.AddField(
+            model_name='dochistoryauthor',
+            name='person',
+            field=models.ForeignKey(blank=True, to='person.Person', null=True),
+        ),
+        migrations.AddField(
+            model_name='documentauthor',
+            name='affiliation',
+            field=models.CharField(help_text=b'Organization/company used by author for submission', max_length=100, blank=True),
+        ),
+        migrations.AddField(
+            model_name='documentauthor',
+            name='country',
+            field=models.CharField(blank=True, help_text=b'Country used by author for submission', max_length=255),
+        ),
+        migrations.RenameField(
+            model_name='documentauthor',
+            old_name='author',
+            new_name='email',
+        ),
+        migrations.AlterField(
+            model_name='documentauthor',
+            name='email',
+            field=models.ForeignKey(blank=True, to='person.Email', help_text=b'Email address used by author for submission', null=True),
+        ),
+        migrations.AddField(
+            model_name='documentauthor',
+            name='person',
+            field=models.ForeignKey(blank=True, to='person.Person', null=True),
+        ),
+        migrations.AlterField(
+            model_name='dochistoryauthor',
+            name='document',
+            field=models.ForeignKey(related_name='documentauthor_set', to='doc.DocHistory'),
+        ),
+        migrations.AlterField(
+            model_name='dochistoryauthor',
+            name='order',
+            field=models.IntegerField(default=1),
+        ),
+        migrations.RunSQL("update doc_documentauthor a inner join person_email e on a.email_id = e.address set a.person_id = e.person_id;", migrations.RunSQL.noop),
+        migrations.RunSQL("update doc_dochistoryauthor a inner join person_email e on a.email_id = e.address set a.person_id = e.person_id;", migrations.RunSQL.noop),
+        migrations.AlterField(
+            model_name='documentauthor',
+            name='person',
+            field=models.ForeignKey(to='person.Person'),
+        ),
+        migrations.AlterField(
+            model_name='dochistoryauthor',
+            name='person',
+            field=models.ForeignKey(to='person.Person'),
+        ),
+    ]

ietf/doc/migrations/0027_remove_fake_email_adresses.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+def fix_invalid_emails(apps, schema_editor):
+    Email = apps.get_model("person", "Email")
+    Role = apps.get_model("group", "Role")
+    RoleHistory = apps.get_model("group", "RoleHistory")
+    DocumentAuthor = apps.get_model("doc", "DocumentAuthor")
+    DocHistoryAuthor = apps.get_model("doc", "DocHistoryAuthor")
+
+    e = Email.objects.filter(address="unknown-email-Gigi-Karmous-Edwards").first()
+    if e:
+        # according to ftp://ietf.org/ietf/97dec/adsl-minutes-97dec.txt
+        new_e, _ = Email.objects.get_or_create(
+            address="GiGi.Karmous-Edwards@pulse.com",
+            primary=e.primary,
+            active=e.active,
+            person=e.person,
+        )
+        Role.objects.filter(email=e).update(email=new_e)
+        RoleHistory.objects.filter(email=e).update(email=new_e)
+        e.delete()
+
+    e = Email.objects.filter(address="unknown-email-Pat-Thaler").first()
+    if e:
+        # current chair email
+        new_e = Email.objects.get(address="pat.thaler@broadcom.com")
+        Role.objects.filter(email=e).update(email=new_e)
+        RoleHistory.objects.filter(email=e).update(email=new_e)
+        e.delete()
+
+    e = Email.objects.filter(address="unknown-email-Greg-<gregimirsky@gmail.com>>").first()
+    if e:
+        # current email
+        new_e = Email.objects.get(address="gregimirsky@gmail.com")
+        DocumentAuthor.objects.filter(email=e).update(email=new_e)
+        DocHistoryAuthor.objects.filter(email=e).update(email=new_e)
+        e.delete()
+
+    DocumentAuthor.objects.filter(email__address__startswith="unknown-email-").exclude(email__address__contains="@").update(email=None)
+    DocHistoryAuthor.objects.filter(email__address__startswith="unknown-email-").exclude(email__address__contains="@").update(email=None)
+    Email.objects.exclude(address__contains="@").filter(address__startswith="unknown-email-").delete()
+
+    assert not Email.objects.filter(address__startswith="unknown-email-")
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('doc', '0026_author_revamp_and_extra_attributes'),
+        ('person', '0014_auto_20160613_0751'),
+        ('group', '0009_auto_20150930_0758'),
+    ]
+
+    operations = [
+        migrations.RunPython(fix_invalid_emails, migrations.RunPython.noop),
+    ]