Added a new personal event table to keep track of personal API key logins, and a management command to send out reports about activity to users with API keys. Added a weekly cronjob script to trigger weekly reports, and a monthly script for future use. Added a @require_api_key decorator to validate API keys for API key views and log in the API key owner. Modified the API key management urls to use create and disable rather than add and delete. Updated the API key list view. Added an API placeholder view function for ballot position setting, for test purposes. Added tests for the decorator and management command.

 - Legacy-Id: 14426

Author: levkowetz

bin/monthly

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Weekly datatracker jobs.
+# 
+# This script is expected to be triggered by cron from
+# /etc/cron.d/datatracker
+
+DTDIR=/a/www/ietf-datatracker/web
+cd $DTDIR/
+
+# Set up the virtual environment
+source $DTDIR/env/bin/activate
+
+logger -p user.info -t cron "Running $DTDIR/bin/monthly"
+

bin/weekly

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Weekly datatracker jobs.
+# 
+# This script is expected to be triggered by cron from
+# /etc/cron.d/datatracker
+
+DTDIR=/a/www/ietf-datatracker/web
+cd $DTDIR/
+
+# Set up the virtual environment
+source $DTDIR/env/bin/activate
+
+logger -p user.info -t cron "Running $DTDIR/bin/weekly"
+
+
+# Send out weekly summaries of apikey usage
+
+$DTDIR/ietf/manage.py send_apikey_usage_emails
+

ietf/api/urls.py

@@ -3,6 +3,7 @@
 from django.conf.urls import include
 
 from ietf import api
+from ietf.doc import views_ballot
 from ietf.meeting import views as meeting_views
 from ietf.submit import views as submit_views
 from ietf.utils.urls import url
@@ -15,6 +16,7 @@
     # Custom API endpoints
     url(r'^notify/meeting/import_recordings/(?P<number>[a-z0-9-]+)/?$', meeting_views.api_import_recordings),
     url(r'^submit/?$', submit_views.api_submit),
+    url(r'^iesg/position', views_ballot.api_set_position),
 ]
 # Additional (standard) Tastypie endpoints
 for n,a in api._api_list:

ietf/doc/views_ballot.py

@@ -3,12 +3,14 @@
 
 import datetime, json
 
-from django.http import HttpResponseForbidden, HttpResponseRedirect, Http404
-from django.shortcuts import render, get_object_or_404, redirect
-from django.urls import reverse as urlreverse
-from django.template.loader import render_to_string
 from django import forms
 from django.conf import settings
+from django.http import HttpResponse, HttpResponseForbidden, HttpResponseRedirect, Http404
+from django.shortcuts import render, get_object_or_404, redirect
+from django.template.loader import render_to_string
+from django.urls import reverse as urlreverse
+from django.views.decorators.csrf import csrf_exempt
+
 
 import debug                            # pyflakes:ignore
 
@@ -23,12 +25,13 @@
 from ietf.doc.lastcall import request_last_call
 from ietf.iesg.models import TelechatDate
 from ietf.ietfauth.utils import has_role, role_required, is_authorized_in_doc_stream
+from ietf.mailtrigger.utils import gather_address_lists
+from ietf.mailtrigger.forms import CcSelectForm
 from ietf.message.utils import infer_message
 from ietf.name.models import BallotPositionName
 from ietf.person.models import Person
 from ietf.utils.mail import send_mail_text, send_mail_preformatted
-from ietf.mailtrigger.utils import gather_address_lists
-from ietf.mailtrigger.forms import CcSelectForm
+from ietf.utils.decorators import require_user_api_key
 
 BALLOT_CHOICES = (("yes", "Yes"),
                   ("noobj", "No Objection"),
@@ -233,6 +236,12 @@ def edit_position(request, name, ballot_id):
                                    blocking_positions=json.dumps(blocking_positions),
                                    ))
 
+@require_user_api_key
+@role_required('Area Director', 'Secretariat')
+@csrf_exempt
+def api_set_position(request):
+    return HttpResponse("Done", status=200, content_type='text/plain')
+
 
 @role_required('Area Director','Secretariat')
 def send_ballot_comment(request, name, ballot_id):

ietf/ietfauth/management/commands/send_apikey_usage_emails.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# Copyright The IETF Trust 2017, All Rights Reserved
+from __future__ import print_function, unicode_literals
+
+import datetime
+
+from textwrap import dedent
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+import debug                            # pyflakes:ignore
+
+from ietf.person.models import PersonalApiKey, PersonApiKeyEvent
+from ietf.utils.mail import send_mail
+
+
+class Command(BaseCommand):
+    """
+    Send out emails to all persons who have personal API keys about usage.
+
+    Usage is show over the given period, where the default period is 7 days.
+    """
+
+    help = dedent(__doc__).strip()
+            
+    def add_arguments(self, parser):
+        parser.add_argument('-d', '--days', dest='days', type=int, default=7,
+            help='The period over which to show usage.')
+
+    def handle(self, *filenames, **options):
+        """
+        """
+
+        self.verbosity = int(options.get('verbosity'))
+        days = options.get('days')
+
+        keys = PersonalApiKey.objects.filter(valid=True)
+        for key in keys:
+            earliest = datetime.datetime.now() - datetime.timedelta(days=days)
+            events = PersonApiKeyEvent.objects.filter(key=key, time__gt=earliest)
+            count = events.count()
+            events = events[:32]
+            if count:
+                key_name = key.hash()[:8]
+                subject = "API key usage for key '%s' for the last %s days" %(key_name, days)
+                to = key.person.email_address()
+                frm = settings.DEFAULT_FROM_EMAIL
+                send_mail(None, to, frm, subject, 'utils/apikey_usage_report.txt',  {'person':key.person,
+                    'days':days, 'key':key, 'key_name':key_name, 'count':count, 'events':events, } )
+                

ietf/ietfauth/tests.py

@@ -17,9 +17,10 @@
 from ietf.utils.test_data import make_test_data, make_review_data
 from ietf.utils.mail import outbox, empty_outbox
 from ietf.group.models import Group, Role, RoleName
+from ietf.group.factories import GroupFactory
 from ietf.ietfauth.htpasswd import update_htpasswd_file
 from ietf.mailinglists.models import Subscribed
-from ietf.person.models import Person, Email
+from ietf.person.models import Person, Email, PersonalApiKey, PERSON_API_KEY_ENDPOINTS
 from ietf.person.factories import PersonFactory
 from ietf.review.models import ReviewWish, UnavailablePeriod
 from ietf.utils.decorators import skip_coverage
@@ -497,7 +498,7 @@ def test_change_username(self):
         self.assertEqual(prev, user)
         self.assertTrue(user.check_password(u'password'))
 
-    def test_apikey(self):
+    def test_apikey_management(self):
         person = PersonFactory()
 
         url = urlreverse('ietf.ietfauth.views.apikey_index')
@@ -511,33 +512,33 @@ def test_apikey(self):
         self.assertContains(r, 'Get a new personal API key')
 
         # Check the add key form content
-        url = urlreverse('ietf.ietfauth.views.apikey_add')
+        url = urlreverse('ietf.ietfauth.views.apikey_create')
         r = self.client.get(url)
         self.assertContains(r, 'Create a new personal API key')
         self.assertContains(r, 'Endpoint')
 
         # Add 2 keys
-        r = self.client.post(url, {'endpoint': '/api/submit'})
-        self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index'))
-        r = self.client.post(url, {'endpoint': '/api/iesg/discuss'})
-        self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index'))
+        for endpoint, display in PERSON_API_KEY_ENDPOINTS:
+            r = self.client.post(url, {'endpoint': endpoint})
+            self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index'))
 
         # Check api key list content
         url = urlreverse('ietf.ietfauth.views.apikey_index')
         r = self.client.get(url)
-        self.assertContains(r, '/api/submit')
-        self.assertContains(r, '/api/iesg/discuss')
+        for endpoint, display in PERSON_API_KEY_ENDPOINTS:
+            self.assertContains(r, endpoint)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('td code')), 2)
+        self.assertEqual(len(q('td code')), len(PERSON_API_KEY_ENDPOINTS)) # hash
+        self.assertEqual(len(q('td a:contains("Disable")')), len(PERSON_API_KEY_ENDPOINTS))
 
         # Get one of the keys
         key = person.apikeys.first()
 
-        # Check the delete key form content
-        url = urlreverse('ietf.ietfauth.views.apikey_del')
+        # Check the disable key form content
+        url = urlreverse('ietf.ietfauth.views.apikey_disable')
         r = self.client.get(url)
 
-        self.assertContains(r, 'Delete a personal API key')
+        self.assertContains(r, 'Disable a personal API key')
         self.assertContains(r, 'Key')
 
         # Delete a key
@@ -547,7 +548,98 @@ def test_apikey(self):
         # Check the api key list content again
         url = urlreverse('ietf.ietfauth.views.apikey_index')
         r = self.client.get(url)
-        self.assertNotContains(r, key.endpoint)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('td code')), 1)
+        self.assertEqual(len(q('td code')), len(PERSON_API_KEY_ENDPOINTS)) # key hash
+        self.assertEqual(len(q('td a:contains("Disable")')), len(PERSON_API_KEY_ENDPOINTS)-1)
+
+    def test_apikey_usage(self):
+        BAD_KEY = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+
+        person = PersonFactory()
+        area = GroupFactory(type_id='area')
+        area.role_set.create(name_id='ad', person=person, email=person.email())
+
+        url = urlreverse('ietf.ietfauth.views.apikey_create')
+        # Check that the url is protected, then log in
+        login_testing_unauthorized(self, person.user.username, url)
+
+        # Add keys
+        for endpoint, display in PERSON_API_KEY_ENDPOINTS:
+            r = self.client.post(url, {'endpoint': endpoint})
+            self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index'))
+
+        for key in person.apikeys.all()[:3]:
+            url = key.endpoint
+
+            # successful access
+            r = self.client.post(url, {'apikey':key.hash(), 'dummy':'dummy',})
+            self.assertEqual(r.status_code, 200)
+
+            # bad method
+            r = self.client.put(url, {'apikey':key.hash()})
+            self.assertEqual(r.status_code, 405)
+
+            # missing apikey
+            r = self.client.post(url, {'dummy':'dummy',})
+            self.assertEqual(r.status_code, 400)
+            self.assertIn('Missing apikey parameter', unicontent(r))
+
+            # invalid apikey
+            r = self.client.post(url, {'apikey':BAD_KEY, 'dummy':'dummy',})
+            self.assertEqual(r.status_code, 400)
+            self.assertIn('Invalid apikey', unicontent(r))
+
+            # too long since regular login
+            person.user.last_login = datetime.datetime.now() - datetime.timedelta(days=settings.UTILS_APIKEY_GUI_LOGIN_LIMIT_DAYS+1)
+            person.user.save()
+            r = self.client.post(url, {'apikey':key.hash(), 'dummy':'dummy',})
+            self.assertEqual(r.status_code, 400)
+            self.assertIn('Too long since last regular login', unicontent(r))
+            person.user.last_login = datetime.datetime.now()
+            person.user.save()
+
+            # endpoint mismatch
+            key2 = PersonalApiKey.objects.create(person=person, endpoint='/')
+            r = self.client.post(url, {'apikey':key2.hash(), 'dummy':'dummy',})
+            self.assertEqual(r.status_code, 400)
+            self.assertIn('Apikey endpoint mismatch', unicontent(r))
+            key2.delete()
+
+    def test_send_apikey_report(self):
+        from ietf.ietfauth.management.commands.send_apikey_usage_emails import Command
+        from ietf.utils.mail import outbox, empty_outbox
+
+        person = PersonFactory()
+
+        url = urlreverse('ietf.ietfauth.views.apikey_create')
+        # Check that the url is protected, then log in
+        login_testing_unauthorized(self, person.user.username, url)
+
+        # Add keys
+        for endpoint, display in PERSON_API_KEY_ENDPOINTS:
+            r = self.client.post(url, {'endpoint': endpoint})
+            self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index'))
+        
+        # Use the endpoints (the form content will not be acceptable, but the
+        # apikey usage will be registered)
+        count = 2
+        # avoid usage across dates
+        if datetime.datetime.now().time() > datetime.time(hour=23, minute=59, second=58):
+            time.sleep(2)
+        for i in range(count):
+            for key in person.apikeys.all():
+                url = key.endpoint
+                self.client.post(url, {'apikey':key.hash(), 'dummy': 'dummy', })
+        date = str(datetime.date.today())
+
+        empty_outbox()
+        cmd = Command()
+        cmd.handle(verbosity=0, days=7)
+        
+        self.assertEqual(len(outbox), len(PERSON_API_KEY_ENDPOINTS))
+        for mail in outbox:
+            body = mail.get_payload()
+            self.assertIn("API key usage", mail['subject'])
+            self.assertIn(" %s times" % count, body)
+            self.assertIn(date, body)
 

ietf/ietfauth/urls.py

@@ -8,8 +8,8 @@
 urlpatterns = [
         url(r'^$', views.index),
         url(r'^apikey/?$', views.apikey_index),
-        url(r'^apikey/add/?$', views.apikey_add),
-        url(r'^apikey/del/?$', views.apikey_del),
+        url(r'^apikey/add/?$', views.apikey_create),
+        url(r'^apikey/del/?$', views.apikey_disable),
         url(r'^confirmnewemail/(?P<auth>[^/]+)/$', views.confirm_new_email),
         url(r'^create/$', views.create_account),
         url(r'^create/confirm/(?P<auth>[^/]+)/$', views.confirm_account),

ietf/ietfauth/views.py

@@ -602,7 +602,7 @@ def apikey_index(request):
 
 @login_required
 @person_required
-def apikey_add(request):
+def apikey_create(request):
     class ApiKeyForm(forms.ModelForm):
         class Meta:
             model = PersonalApiKey
@@ -623,7 +623,7 @@ class Meta:
 
 @login_required
 @person_required
-def apikey_del(request):
+def apikey_disable(request):
     person = request.user.person
     choices = [ (k.hash(), str(k)) for k in person.apikeys.all() ]
     #
@@ -642,11 +642,12 @@ def clean_key(self):
         if form.is_valid():
             hash = form.data['hash']
             key = PersonalApiKey.validate_key(hash)
-            key.delete()
-            messages.success(request, "Deleted key %s" % hash)
+            key.valid = False
+            key.save()
+            messages.success(request, "Disabled key %s" % hash)
             return redirect('ietf.ietfauth.views.apikey_index')
         else:
-            messages.error(request, "Key validation failed; key not deleted")
+            messages.error(request, "Key validation failed; key not disabled")
     else:
         form = KeyDeleteForm(request.GET)
-    return render(request, 'form.html', {'form':form, 'title':"Delete a personal API key", 'description':'', 'button':'Delete key'})
+    return render(request, 'form.html', {'form':form, 'title':"Disable a personal API key", 'description':'', 'button':'Disable key'})

ietf/person/admin.py

@@ -1,7 +1,7 @@
 from django.contrib import admin
 
 
-from ietf.person.models import Email, Alias, Person, PersonHistory, PersonalApiKey
+from ietf.person.models import Email, Alias, Person, PersonHistory, PersonalApiKey, PersonEvent, PersonApiKeyEvent
 from ietf.person.name import name_parts
 
 class EmailAdmin(admin.ModelAdmin):
@@ -49,3 +49,15 @@ class PersonalApiKeyAdmin(admin.ModelAdmin):
     raw_id_fields = ['person', ]
     search_fields = ['person__name', ]
 admin.site.register(PersonalApiKey, PersonalApiKeyAdmin)
+
+class PersonEventAdmin(admin.ModelAdmin):
+    list_display = ["id", "person", "time", "type", ]
+    search_fields = ["person__name", ]
+    raw_id_fields = ['person', ]
+admin.site.register(PersonEvent, PersonEventAdmin)
+
+class PersonApiKeyEventAdmin(admin.ModelAdmin):
+    list_display = ["id", "person", "time", "type", "key"]
+    search_fields = ["person__name", ]
+    raw_id_fields = ['person', ]
+admin.site.register(PersonApiKeyEvent, PersonApiKeyEventAdmin)