Merged in the latest GDPR changes. This refines the handling of the consent checkbox on the account page; refines the Consent Needed warning given on login if consent is needed; tweaks several models to set the on_deletion fields for FK to User and Person appropriately; adds a Person.needs_consent() method to capture the logic of which fields require consent; refines the Person.plain_name() method and the user.log.log() function; and adds 2 management commands to send out consent requests and delete non-consent information, respectively.

 - Legacy-Id: 15464

Author: levkowetz

ietf/ietfauth/forms.py

@@ -116,8 +116,6 @@ def __init__(self, *args, **kwargs):
                 if f in self.fields:
                     self.fields[f].label += ' \u2020'
 
-            self.fields["consent"].required = True
-
             self.unidecoded_ascii = False
 
             if self.data and not self.data.get("ascii", "").strip():
@@ -150,8 +148,13 @@ def clean_ascii_short(self):
 
         def clean_consent(self):
             consent = self.cleaned_data.get('consent')
-            if consent == False:
-                raise forms.ValidationError("In order to modify your profile data, you must permit the IETF to use the uploaded data.")
+            require_consent = (
+                self.cleaned_data.get('name') != person.name_from_draft
+                or self.cleaned_data.get('ascii') != person.name_from_draft
+                or self.cleaned_data.get('biography')
+            )
+            if consent == False and require_consent:
+                raise forms.ValidationError("In order to modify your profile with data that require consent, you must permit the IETF to use the uploaded data.")
             return consent
 
     return PersonForm(*args, **kwargs)

ietf/ietfauth/views.py

@@ -580,12 +580,14 @@ def login(request, extra_context=None):
     which is not recognized as a valid password hash.
     """
 
+    require_consent = []
     if request.method == "POST":
         form = AuthenticationForm(request, data=request.POST)
         username = form.data.get('username')
         user = User.objects.filter(username=username).first()
         #
-        require_consent = []
+        if user.person and not user.person.consent:
+            require_consent = user.person.needs_consent()
         if user:
             if hasattr(user, 'person') and not user.person.consent:
                 person = user.person
@@ -618,8 +620,10 @@ def login(request, extra_context=None):
                 You have personal information associated with your account which is not
                 derived from draft submissions or other ietf work, namely: %s.  Please go
                 to your <a href='/accounts/profile'>account profile</a> and review your
-                personal information, and confirm that it may be used and displayed
-                within the IETF datatracker.
+                personal information, then scoll to the bottom and check the 'confirm'
+                checkbox and submit the form, in order to to indicate that that the
+                provided personal information may be used and displayed within the IETF
+                datatracker.
 
                 """ % ', '.join(require_consent)))
     return response

ietf/nomcom/models.py

@@ -90,7 +90,7 @@ class Nomination(models.Model):
     nominee = ForeignKey('Nominee')
     comments = ForeignKey('Feedback')
     nominator_email = models.EmailField(verbose_name='Nominator Email', blank=True)
-    user = ForeignKey(User, editable=False)
+    user = ForeignKey(User, editable=False, null=True, on_delete=models.SET_NULL)
     time = models.DateTimeField(auto_now_add=True)
     share_nominator = models.BooleanField(verbose_name='Share nominator name with candidate', default=False,
                                           help_text='Check this box to allow the NomCom to let the '
@@ -247,7 +247,7 @@ class Feedback(models.Model):
     subject = models.TextField(verbose_name='Subject', blank=True)
     comments = EncryptedTextField(verbose_name='Comments')
     type = ForeignKey(FeedbackTypeName, blank=True, null=True)
-    user = ForeignKey(User, editable=False, blank=True, null=True)
+    user = ForeignKey(User, editable=False, blank=True, null=True, on_delete=models.SET_NULL)
     time = models.DateTimeField(auto_now_add=True)
 
     objects = FeedbackManager()

ietf/person/migrations/0006_auto_20180910_0719.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-09-10 07:19
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations
+import django.db.models.deletion
+import ietf.utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('person', '0005_populate_person_name_from_draft'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='person',
+            name='user',
+            field=ietf.utils.models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL),
+        ),
+    ]

ietf/person/models.py

@@ -12,6 +12,7 @@
 from django.conf import settings
 
 from django.core.validators import validate_email
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.contrib.auth.models import User
 from django.template.loader import render_to_string
@@ -31,7 +32,7 @@
 
 class Person(models.Model):
     history = HistoricalRecords()
-    user = OneToOneField(User, blank=True, null=True)
+    user = OneToOneField(User, blank=True, null=True, on_delete=models.SET_NULL)
     time = models.DateTimeField(default=datetime.datetime.now)      # When this Person record entered the system
     # The normal unicode form of the name.  This must be
     # set to the same value as the ascii-form if equal.
@@ -163,6 +164,31 @@ def expired_drafts(self):
         from ietf.doc.models import Document
         return Document.objects.filter(documentauthor__person=self, type='draft', states__slug__in=['repl', 'expired', 'auth-rm', 'ietf-rm']).order_by('-time')
 
+    def needs_consent(self):
+        """
+        Returns an empty list or a list of fields which holds information that
+        requires consent to be given.
+        """
+        needs_consent = []
+        if self.name != self.name_from_draft:
+            needs_consent.append("full name")
+        if self.ascii != self.name_from_draft:
+            needs_consent.append("ascii name")
+        if self.biography and not (self.role_set.exists() or self.rolehistory_set.exists()):
+            needs_consent.append("biography")
+        if self.user_id:
+            needs_consent.append("login")
+            try:
+                if self.user.communitylist_set.exists():
+                    needs_consent.append("draft notification subscription(s)")
+            except ObjectDoesNotExist:
+                pass
+        for email in self.email_set.all():
+            if not email.origin.split(':')[0] in ['author', 'role', 'reviewer', 'liaison', 'shepherd', ]:
+                needs_consent.append("email address(es)")
+                break
+        return needs_consent
+
     def save(self, *args, **kwargs):
         created = not self.pk
         super(Person, self).save(*args, **kwargs)

ietf/person/name.py

@@ -71,7 +71,7 @@ def initials(name):
 
 def plain_name(name):
     prefix, first, middle, last, suffix = name_parts(name)
-    return u" ".join([first, last])
+    return u" ".join( n for n in (first, last) if n)
 
 def capfirst(s):
     # Capitalize the first word character, skipping non-word characters and

ietf/templates/utils/personal_information_notice.txt

@@ -0,0 +1,23 @@
+{% load ietf_filters %}{% filter wordwrap:78 %}
+Dear {{ person.plain_name }},
+
+This email is regarding some of the personal information stored in your IETF datatracker
+profile; information for which we require your consent for storage and use.
+
+If you do nothing in response to this email, the information in your profile
+that requires consent ({{ fields|safe }} and login) will be deleted {{ days }}
+days from now, on {{ date }}.  If you later wish to create a new login, you can
+do so at {{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.ietfauth.views.create_account' %}.
+
+If you would like to keep the information that requires consent available, please go to
+{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.ietfauth.views.profile' %}, and review and
+edit the information as desired.  When ready, please check the 'Consent' checkbox found
+at the bottom of the page and submit the form. 
+
+For information on how personal information is handled in the datatracker, please see
+{{ settings.IDTRACKER_BASE_URL }}/help/personal-information.
+
+
+Thank You,
+The IETF Secretariat
+{% endfilter %}

ietf/utils/log.py

@@ -43,7 +43,7 @@ def _flushfunc():
         return
     elif settings.DEBUG == True:
         _logfunc = debug.say
-        _flushfunc = sys.stdout.flush
+        _flushfunc = sys.stdout.flush   # pyflakes:ignore (intentional redefinition)
     if isinstance(msg, unicode):
         msg = msg.encode('unicode_escape')
     try:

ietf/utils/management/commands/delete_data_lacking_consent.py

@@ -0,0 +1,174 @@
+# Copyright The IETF Trust 2016, All Rights Reserved
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals, print_function
+
+import datetime
+from tqdm import tqdm
+
+from django.conf import settings
+from django.contrib.admin.utils import NestedObjects
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand
+from django.db.models import F
+
+import debug                            # pyflakes:ignore
+
+from ietf.community.models import SearchRule
+from ietf.person.models import Person, Alias, PersonalApiKey, Email
+from ietf.person.name import unidecode_name
+from ietf.utils.log import log
+
+class Command(BaseCommand):
+    help = (u"""
+
+        Delete data for which consent to store the data has not been given,
+        where the data does not fall under the GDPR Legitimate Interest clause
+        for the IETF.  This includes full name, ascii name, bio, login,
+        notification subscriptions and email addresses that are not derived from
+        published drafts or ietf roles.
+
+        """)
+
+    def add_arguments(self, parser):
+         parser.add_argument('-n', '--dry-run', action='store_true', default=False,
+             help="Don't delete anything, just list what would be done.")
+#          parser.add_argument('-d', '--date', help="Date of deletion (mentioned in message)")
+         parser.add_argument('-m', '--minimum-response-time', metavar='TIME', type=int, default=14,
+             help="Minimum response time, default: %(default)s days.  Persons to whom a "
+                  "consent request email has been sent more recently than this will not "
+                  "be affected by the run.")
+#          parser.add_argument('-r', '--rate', type=float, default=1.0,
+#              help='Rate of sending mail, default: %(default)s/s')
+#          parser.add_argument('user', nargs='*')
+         
+
+    def handle(self, *args, **options):
+        dry_run    = options['dry_run']
+        verbosity  = int(options['verbosity'])
+        event_type = 'gdpr_notice_email'
+        settings.DEBUG = False          # don't log to console
+
+        # users
+        users = User.objects.filter(person__isnull=True, username__contains='@')
+        self.stdout.write("Found %d users without associated person records" % (users.count(), ))
+        emails = Email.objects.filter(address__in=users.values_list('username', flat=True))
+        # fix up users that don't have person records, but have a username matching a nown email record
+        self.stdout.write("Checking usernames against email records ...")
+        for email in tqdm(emails):
+            user = users.get(username=email.address)
+            if email.person.user_id:
+                if dry_run:
+                    self.stdout.write("Would delete user  #%-6s (%s) %s" % (user.id, user.last_login, user.username))
+                else:
+                    log("Deleting user #%-6s (%s) %s: no person record, matching email has other user" % (user.id, user.last_login, user.username))
+                    user_id = user.id
+                    user.delete()
+                    Person.history.filter(user_id=user_id).delete()
+                    Email.history.filter(history_user=user_id).delete()
+            else:
+                if dry_run:
+                    self.stdout.write("Would connect user #%-6s %s to person #%-6s %s" % (user.id, user.username, email.person.id, email.person.ascii_name()))
+                else:
+                    log("Connecting user #%-6s %s to person #%-6s %s" % (user.id, user.username, email.person.id, email.person.ascii_name()))
+                    email.person.user_id = user.id
+                    email.person.save()
+        # delete users without person records
+        users = users.exclude(username__in=emails.values_list('address', flat=True))
+        if dry_run:
+            self.stdout.write("Would delete %d users without associated person records" % (users.count(), ))
+        else:
+            if users.count():
+                log("Deleting %d users without associated person records" % (users.count(), ))
+                assert not users.filter(person__isnull=False).exists()
+                user_ids = users.values_list('id', flat=True)
+                users.delete()
+                assert not Person.history.filter(user_id__in=user_ids).exists()
+
+
+        # persons
+        self.stdout.write('Querying the database for person records without given consent ...')
+        notification_cutoff = datetime.datetime.now() - datetime.timedelta(days=options['minimum_response_time'])
+        persons = Person.objects.exclude(consent=True)
+        persons = persons.exclude(id=1) # make sure we don't delete System ;-)
+        self.stdout.write("Found %d persons with information for which we don't have consent." % (persons.count(), ))
+
+        # Narrow to persons we don't have Legitimate Interest in, and delete those fully
+        persons = persons.exclude(docevent__by=F('pk'))
+        persons = persons.exclude(documentauthor__person=F('pk')).exclude(dochistoryauthor__person=F('pk'))
+        persons = persons.exclude(email__liaisonstatement__from_contact__person=F('pk'))
+        persons = persons.exclude(email__reviewrequest__reviewer__person=F('pk'))
+        persons = persons.exclude(email__shepherd_dochistory_set__shepherd__person=F('pk'))
+        persons = persons.exclude(email__shepherd_document_set__shepherd__person=F('pk'))
+        persons = persons.exclude(iprevent__by=F('pk'))
+        persons = persons.exclude(meetingregistration__person=F('pk'))
+        persons = persons.exclude(message__by=F('pk'))
+        persons = persons.exclude(name_from_draft='')
+        persons = persons.exclude(personevent__time__gt=notification_cutoff, personevent__type=event_type)
+        persons = persons.exclude(reviewrequest__requested_by=F('pk'))
+        persons = persons.exclude(role__person=F('pk')).exclude(rolehistory__person=F('pk'))
+        persons = persons.exclude(session__requested_by=F('pk'))
+        persons = persons.exclude(submissionevent__by=F('pk'))
+        self.stdout.write("Found %d persons with information for which we neither have consent nor legitimate interest." % (persons.count(), ))
+        if persons.count() > 0:
+            self.stdout.write("Deleting records for persons for which we have with neither consent nor legitimate interest ...")
+            for person in (persons if dry_run else tqdm(persons)):
+                if dry_run:
+                    self.stdout.write(("Would delete record   #%-6d: (%s) %-32s %-48s" % (person.pk, person.time, person.ascii_name(), "<%s>"%person.email())).encode('utf8'))
+                else:
+                    if verbosity > 1:
+                        # development aids
+                        collector = NestedObjects(using='default')
+                        collector.collect([person,])
+                        objects = collector.nested()
+                        related = [ o for o in objects[-1] if not isinstance(o, (Alias, Person, SearchRule, PersonalApiKey)) ]
+                        if len(related) > 0:
+                            self.stderr.write("Person record #%-6s %s has unexpected related records" % (person.pk, person.ascii_name()))
+
+                    # Historical records using simple_history has on_delete=DO_NOTHING, so
+                    # we have to do explicit deletions:
+                    id = person.id
+                    person.delete()
+                    Person.history.filter(id=id).delete()
+                    Email.history.filter(person_id=id).delete()
+
+        # Deal with remaining persons (lacking consent, but with legitimate interest)
+        persons = Person.objects.exclude(consent=True)
+        persons = persons.exclude(id=1)
+        self.stdout.write("Found %d remaining persons with information for which we don't have consent." % (persons.count(), ))
+        if persons.count() > 0:
+            self.stdout.write("Removing personal information requiring consent ...")
+            for person in (persons if dry_run else tqdm(persons)):
+                fields = ', '.join(person.needs_consent())
+                if dry_run:
+                    self.stdout.write(("Would remove info for #%-6d: (%s) %-32s %-48s %s" % (person.pk, person.time, person.ascii_name(), "<%s>"%person.email(), fields)).encode('utf8'))
+                else:
+                    if person.name_from_draft:
+                        log("Using name info from draft for #%-6d %s: no consent, no roles" % (person.pk, person))
+                        person.name = person.name_from_draft
+                        person.ascii = unidecode_name(person.name_from_draft)
+                    if person.biography:
+                        log("Deleting biography for #%-6d %s: no consent, no roles" % (person.pk, person))
+                        person.biography = ''
+                        person.save()
+                    if person.user_id:
+                        if User.objects.filter(id=person.user_id).exists():
+                            log("Deleting communitylist for #%-6d %s: no consent, no roles" % (person.pk, person))
+                            person.user.communitylist_set.all().delete()
+                    for email in person.email_set.all():
+                        if not email.origin.split(':')[0] in ['author', 'role', 'reviewer', 'liaison', 'shepherd', ]:
+                            log("Deleting email <%s> for #%-6d %s: no consent, no roles" % (email.address, person.pk, person))
+                            address = email.address
+                            email.delete()
+                            Email.history.filter(address=address).delete()
+
+        emails = Email.objects.filter(origin='', person__consent=False)
+        self.stdout.write("Found %d emails without origin for which we lack consent." % (emails.count(), ))
+        if dry_run:
+            self.stdout.write("Would delete %d email records without origin and consent" % (emails.count(), ))
+        else:
+            if emails.count():
+                log("Deleting %d email records without origin and consent" % (emails.count(), ))
+                addresses = emails.values_list('address', flat=True)
+                emails.delete()
+                Email.history.filter(address__in=addresses).delete()
+