From ca88cb19d6fe51116ff40f860dc86311ab714bcb Mon Sep 17 00:00:00 2001 From: Ryan Cross Date: Sat, 19 Jul 2025 12:46:40 +0200 Subject: [PATCH] refactor: remove old reg API functions --- ietf/api/tests.py | 126 ----------------- ietf/api/urls.py | 1 - ietf/api/views.py | 95 ------------- ietf/meeting/tasks.py | 18 --- ietf/meeting/tests_utils.py | 89 +----------- ietf/meeting/utils.py | 112 --------------- ietf/settings.py | 1 - .../commands/fetch_meeting_attendance.py | 42 ------ ietf/stats/tasks.py | 27 ---- ietf/stats/tests.py | 124 ----------------- ietf/stats/utils.py | 127 +----------------- 11 files changed, 3 insertions(+), 759 deletions(-) delete mode 100644 ietf/stats/management/commands/fetch_meeting_attendance.py delete mode 100644 ietf/stats/tasks.py diff --git a/ietf/api/tests.py b/ietf/api/tests.py index abaf9f5ed2..93a2195467 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -37,7 +37,6 @@ from ietf.nomcom.factories import NomComFactory, nomcom_kwargs_for_year from ietf.person.factories import PersonFactory, random_faker, EmailFactory, PersonalApiKeyFactory from ietf.person.models import Email, User -from ietf.stats.models import MeetingRegistration from ietf.utils.mail import empty_outbox, outbox, get_payload_text from ietf.utils.models import DumpInfo from ietf.utils.test_utils import TestCase, login_testing_unauthorized, reload_db_objects @@ -705,131 +704,6 @@ def test_api_v2_person_export_view(self): self.assertEqual(data['ascii'], robot.ascii) self.assertEqual(data['user']['email'], robot.user.email) - def test_api_new_meeting_registration(self): - meeting = MeetingFactory(type_id='ietf') - reg = { - 'apikey': 'invalid', - 'affiliation': "Alguma Corporação", - 'country_code': 'PT', - 'email': 'foo@example.pt', - 'first_name': 'Foo', - 'last_name': 'Bar', - 'meeting': meeting.number, - 'reg_type': 'hackathon', - 'ticket_type': '', - 'checkedin': 'False', - 'is_nomcom_volunteer': 'False', - } - url = urlreverse('ietf.api.views.api_new_meeting_registration') - r = self.client.post(url, reg) - self.assertContains(r, 'Invalid apikey', status_code=403) - oidcp = PersonFactory(user__is_staff=True) - # Make sure 'oidcp' has an acceptable role - RoleFactory(name_id='robot', person=oidcp, email=oidcp.email(), group__acronym='secretariat') - key = PersonalApiKeyFactory(person=oidcp, endpoint=url) - reg['apikey'] = key.hash() - # - # Test valid POST - # FIXME: sometimes, there seems to be something in the outbox? - old_len = len(outbox) - r = self.client.post(url, reg) - self.assertContains(r, "Accepted, New registration, Email sent", status_code=202) - # - # Check outgoing mail - self.assertEqual(len(outbox), old_len + 1) - body = get_payload_text(outbox[-1]) - self.assertIn(reg['email'], outbox[-1]['To'] ) - self.assertIn(reg['email'], body) - self.assertIn('account creation request', body) - # - # Check record - obj = MeetingRegistration.objects.get(email=reg['email'], meeting__number=reg['meeting']) - for key in ['affiliation', 'country_code', 'first_name', 'last_name', 'person', 'reg_type', 'ticket_type', 'checkedin']: - self.assertEqual(getattr(obj, key), False if key=='checkedin' else reg.get(key) , "Bad data for field '%s'" % key) - # - # Test with existing user - person = PersonFactory() - reg['email'] = person.email().address - reg['first_name'] = person.first_name() - reg['last_name'] = person.last_name() - # - r = self.client.post(url, reg) - self.assertContains(r, "Accepted, New registration", status_code=202) - # - # There should be no new outgoing mail - self.assertEqual(len(outbox), old_len + 1) - # - # Test multiple reg types - reg['reg_type'] = 'remote' - reg['ticket_type'] = 'full_week_pass' - r = self.client.post(url, reg) - self.assertContains(r, "Accepted, New registration", status_code=202) - objs = MeetingRegistration.objects.filter(email=reg['email'], meeting__number=reg['meeting']) - self.assertEqual(len(objs), 2) - self.assertEqual(objs.filter(reg_type='hackathon').count(), 1) - self.assertEqual(objs.filter(reg_type='remote', ticket_type='full_week_pass').count(), 1) - self.assertEqual(len(outbox), old_len + 1) - # - # Test incomplete POST - drop_fields = ['affiliation', 'first_name', 'reg_type'] - for field in drop_fields: - del reg[field] - r = self.client.post(url, reg) - self.assertContains(r, 'Missing parameters:', status_code=400) - err, fields = r.content.decode().split(':', 1) - missing_fields = [f.strip() for f in fields.split(',')] - self.assertEqual(set(missing_fields), set(drop_fields)) - - def test_api_new_meeting_registration_nomcom_volunteer(self): - '''Test that Volunteer is created if is_nomcom_volunteer=True - is submitted to API - ''' - meeting = MeetingFactory(type_id='ietf') - reg = { - 'apikey': 'invalid', - 'affiliation': "Alguma Corporação", - 'country_code': 'PT', - 'meeting': meeting.number, - 'reg_type': 'onsite', - 'ticket_type': '', - 'checkedin': 'False', - 'is_nomcom_volunteer': 'False', - } - person = PersonFactory() - reg['email'] = person.email().address - reg['first_name'] = person.first_name() - reg['last_name'] = person.last_name() - now = datetime.datetime.now() - if now.month > 10: - year = now.year + 1 - else: - year = now.year - # create appropriate group and nomcom objects - nomcom = NomComFactory.create(is_accepting_volunteers=True, **nomcom_kwargs_for_year(year)) - url = urlreverse('ietf.api.views.api_new_meeting_registration') - oidcp = PersonFactory(user__is_staff=True) - # Make sure 'oidcp' has an acceptable role - RoleFactory(name_id='robot', person=oidcp, email=oidcp.email(), group__acronym='secretariat') - key = PersonalApiKeyFactory(person=oidcp, endpoint=url) - reg['apikey'] = key.hash() - - # first test is_nomcom_volunteer False - r = self.client.post(url, reg) - self.assertContains(r, "Accepted, New registration", status_code=202) - # assert no Volunteers exists - self.assertEqual(Volunteer.objects.count(), 0) - - # test is_nomcom_volunteer True - reg['is_nomcom_volunteer'] = 'True' - r = self.client.post(url, reg) - self.assertContains(r, "Accepted, Updated registration", status_code=202) - # assert Volunteer exists - self.assertEqual(Volunteer.objects.count(), 1) - volunteer = Volunteer.objects.last() - self.assertEqual(volunteer.person, person) - self.assertEqual(volunteer.nomcom, nomcom) - self.assertEqual(volunteer.origin, 'registration') - @override_settings(APP_API_TOKENS={"ietf.api.views.api_new_meeting_registration_v2": ["valid-token"]}) def test_api_new_meeting_registration_v2(self): meeting = MeetingFactory(type_id='ietf') diff --git a/ietf/api/urls.py b/ietf/api/urls.py index bafd5c5b76..6f2efb3c1e 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -67,7 +67,6 @@ url(r'^notify/session/polls/?$', meeting_views.api_upload_polls), # Let the registration system notify us about registrations url(r'^notify/meeting/registration/v2/?', api_views.api_new_meeting_registration_v2), - url(r'^notify/meeting/registration/?', api_views.api_new_meeting_registration), # OpenID authentication provider url(r'^openid/$', TemplateView.as_view(template_name='api/openid-issuer.html'), name='ietf.api.urls.oidc_issuer'), url(r'^openid/', include('oidc_provider.urls', namespace='oidc_provider')), diff --git a/ietf/api/views.py b/ietf/api/views.py index e8e38b25b4..b4dd7f05d6 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -16,8 +16,6 @@ from django.contrib.auth import authenticate from django.contrib.auth.decorators import login_required from django.contrib.auth.models import User -from django.core.exceptions import ValidationError -from django.core.validators import validate_email from django.http import HttpResponse, Http404, JsonResponse, HttpResponseBadRequest from django.shortcuts import render, get_object_or_404 from django.urls import reverse @@ -43,14 +41,11 @@ from ietf.doc.utils import DraftAliasGenerator, fuzzy_find_documents from ietf.group.utils import GroupAliasGenerator, role_holder_emails from ietf.ietfauth.utils import role_required -from ietf.ietfauth.views import send_account_creation_email from ietf.ipr.utils import ingest_response_email as ipr_ingest_response_email from ietf.meeting.models import Meeting from ietf.meeting.utils import import_registration_json_validator, process_single_registration -from ietf.nomcom.models import Volunteer, NomCom from ietf.nomcom.utils import ingest_feedback_email as nomcom_ingest_feedback_email from ietf.person.models import Person, Email -from ietf.stats.models import MeetingRegistration from ietf.sync.iana import ingest_review_email as iana_ingest_review_email from ietf.utils import log from ietf.utils.decorators import require_api_key @@ -151,96 +146,6 @@ def post(self, request): # else: # return HttpResponse(status=405) -@require_api_key -@role_required('Robot') -@csrf_exempt -def api_new_meeting_registration(request): - '''REST API to notify the datatracker about a new meeting registration''' - def err(code, text): - return HttpResponse(text, status=code, content_type='text/plain') - required_fields = [ 'meeting', 'first_name', 'last_name', 'affiliation', 'country_code', - 'email', 'reg_type', 'ticket_type', 'checkedin', 'is_nomcom_volunteer'] - fields = required_fields + [] - if request.method == 'POST': - # parameters: - # apikey: - # meeting - # name - # email - # reg_type (In Person, Remote, Hackathon Only) - # ticket_type (full_week, one_day, student) - # - data = {'attended': False, } - missing_fields = [] - for item in fields: - value = request.POST.get(item, None) - if value is None and item in required_fields: - missing_fields.append(item) - data[item] = value - if missing_fields: - return err(400, "Missing parameters: %s" % ', '.join(missing_fields)) - number = data['meeting'] - try: - meeting = Meeting.objects.get(number=number) - except Meeting.DoesNotExist: - return err(400, "Invalid meeting value: '%s'" % (number, )) - reg_type = data['reg_type'] - email = data['email'] - try: - validate_email(email) - except ValidationError: - return err(400, "Invalid email value: '%s'" % (email, )) - if request.POST.get('cancelled', 'false') == 'true': - MeetingRegistration.objects.filter( - meeting_id=meeting.pk, - email=email, - reg_type=reg_type).delete() - return HttpResponse('OK', status=200, content_type='text/plain') - else: - object, created = MeetingRegistration.objects.get_or_create( - meeting_id=meeting.pk, - email=email, - reg_type=reg_type) - try: - # Update attributes - for key in set(data.keys())-set(['attended', 'apikey', 'meeting', 'email']): - if key == 'checkedin': - new = bool(data.get(key).lower() == 'true') - else: - new = data.get(key) - setattr(object, key, new) - person = Person.objects.filter(email__address=email) - if person.exists(): - object.person = person.first() - object.save() - except ValueError as e: - return err(400, "Unexpected POST data: %s" % e) - response = "Accepted, New registration" if created else "Accepted, Updated registration" - if User.objects.filter(username__iexact=email).exists() or Email.objects.filter(address=email).exists(): - pass - else: - send_account_creation_email(request, email) - response += ", Email sent" - - # handle nomcom volunteer - if request.POST.get('is_nomcom_volunteer', 'false').lower() == 'true' and object.person: - try: - nomcom = NomCom.objects.get(is_accepting_volunteers=True) - except (NomCom.DoesNotExist, NomCom.MultipleObjectsReturned): - nomcom = None - if nomcom: - Volunteer.objects.get_or_create( - nomcom=nomcom, - person=object.person, - defaults={ - "affiliation": data["affiliation"], - "origin": "registration" - } - ) - return HttpResponse(response, status=202, content_type='text/plain') - else: - return HttpResponse(status=405) - @requires_api_token @csrf_exempt diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py index dc3fbc99ec..784eb00d87 100644 --- a/ietf/meeting/tasks.py +++ b/ietf/meeting/tasks.py @@ -9,7 +9,6 @@ from .models import Meeting from .utils import generate_proceedings_content from .views import generate_agenda_data -from .utils import migrate_registrations, check_migrate_registrations from .utils import fetch_attendance_from_meetings @@ -18,23 +17,6 @@ def agenda_data_refresh(): generate_agenda_data(force_refresh=True) -@shared_task -def migrate_registrations_task(initial=False): - """ Migrate ietf.stats.MeetingRegistration to ietf.meeting.Registration - If initial is True, migrate all meetings otherwise only future meetings. - This function is idempotent. It can be run regularly from cron. - """ - migrate_registrations(initial=initial) - - -@shared_task -def check_migrate_registrations_task(): - """ Compare MeetingRegistration with Registration to ensure - all records migrated - """ - check_migrate_registrations() - - @shared_task def proceedings_content_refresh_task(*, all=False): """Refresh meeting proceedings cache diff --git a/ietf/meeting/tests_utils.py b/ietf/meeting/tests_utils.py index 8d912158ce..11ea63df4f 100644 --- a/ietf/meeting/tests_utils.py +++ b/ietf/meeting/tests_utils.py @@ -12,101 +12,14 @@ from django.http import HttpResponse, JsonResponse from ietf.meeting.factories import MeetingFactory, RegistrationFactory, RegistrationTicketFactory from ietf.meeting.models import Registration -from ietf.meeting.utils import (migrate_registrations, get_preferred, process_single_registration, +from ietf.meeting.utils import (process_single_registration, get_registration_data, sync_registration_data, fetch_attendance_from_meetings) from ietf.nomcom.models import Volunteer from ietf.nomcom.factories import NomComFactory, nomcom_kwargs_for_year from ietf.person.factories import PersonFactory -from ietf.stats.factories import MeetingRegistrationFactory from ietf.utils.test_utils import TestCase -class MigrateRegistrationsTests(TestCase): - def test_new_meeting_registration(self): - meeting = MeetingFactory(type_id='ietf', number='109') - reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass') - self.assertEqual(Registration.objects.count(), 0) - migrate_registrations(initial=True) - self.assertEqual(Registration.objects.count(), 1) - new = Registration.objects.first() - self.assertEqual(new.first_name, reg.first_name) - self.assertEqual(new.last_name, reg.last_name) - self.assertEqual(new.email, reg.email) - self.assertEqual(new.person, reg.person) - self.assertEqual(new.meeting, meeting) - self.assertEqual(new.affiliation, reg.affiliation) - self.assertEqual(new.country_code, reg.country_code) - self.assertEqual(new.checkedin, reg.checkedin) - self.assertEqual(new.attended, reg.attended) - - def test_migrate_non_initial(self): - # with only old meeting - meeting = MeetingFactory(type_id='ietf', number='109') - MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass') - self.assertEqual(Registration.objects.count(), 0) - migrate_registrations() - self.assertEqual(Registration.objects.count(), 0) - # with new meeting - new_meeting = MeetingFactory(type_id='ietf', number='150') - new_meeting.date = datetime.date.today() + datetime.timedelta(days=30) - new_meeting.save() - MeetingRegistrationFactory(meeting=new_meeting, reg_type='onsite', ticket_type='week_pass') - migrate_registrations() - self.assertEqual(Registration.objects.count(), 1) - - def test_updated_meeting_registration(self): - # setup test initial conditions - meeting = MeetingFactory(type_id='ietf', number='109') - reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass') - migrate_registrations(initial=True) - # change first_name and save - original = reg.first_name - reg.first_name = 'NewBob' - reg.save() - new = Registration.objects.first() - self.assertEqual(new.first_name, original) - migrate_registrations(initial=True) - new.refresh_from_db() - self.assertEqual(new.first_name, reg.first_name) - - def test_additional_ticket(self): - # setup test initial conditions - meeting = MeetingFactory(type_id='ietf', number='109') - reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass') - migrate_registrations(initial=True) - new = Registration.objects.first() - self.assertEqual(new.tickets.count(), 1) - # add a second ticket - reg.reg_type = 'remote' - reg.pk = None - reg.save() - migrate_registrations(initial=True) - # new.refresh_from_db() - self.assertEqual(new.tickets.count(), 2) - - def test_cancelled_registration(self): - # setup test initial conditions - meeting = MeetingFactory(type_id='ietf', number='109') - reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass') - migrate_registrations(initial=True) - reg.delete() - # do test - migrate_registrations(initial=True) - self.assertEqual(Registration.objects.count(), 0) - - def test_get_preferred(self): - meeting = MeetingFactory(type_id='ietf', number='109') - onsite = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass') - remote = MeetingRegistrationFactory(meeting=meeting, reg_type='remote', ticket_type='week_pass') - hackathon = MeetingRegistrationFactory(meeting=meeting, reg_type='hackathon_onsite', ticket_type='week_pass') - result = get_preferred([remote, onsite, hackathon]) - self.assertEqual(result, onsite) - result = get_preferred([hackathon, remote]) - self.assertEqual(result, remote) - result = get_preferred([hackathon]) - self.assertEqual(result, hackathon) - - class JsonResponseWithJson(JsonResponse): def json(self): return json.loads(self.content) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index db67f79b93..2c9c117158 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -37,7 +37,6 @@ from ietf.group.utils import can_manage_materials from ietf.name.models import SessionStatusName, ConstraintName, DocTypeName from ietf.person.models import Person -from ietf.stats.models import MeetingRegistration from ietf.utils.html import clean_html from ietf.utils.log import log from ietf.utils.timezone import date_today @@ -1017,117 +1016,6 @@ def participants_for_meeting(meeting): return (checked_in, attended) -def get_preferred(regs): - """ If there are multiple registrations return preferred in - this order: onsite, remote, any (ie hackathon_onsite) - """ - if len(regs) == 1: - return regs[0] - reg_types = [r.reg_type for r in regs] - if 'onsite' in reg_types: - return regs[reg_types.index('onsite')] - elif 'remote' in reg_types: - return regs[reg_types.index('remote')] - else: - return regs[0] - - -def migrate_registrations(initial=False): - """ Migrate ietf.stats.MeetingRegistration to ietf.meeting.Registration - If initial is True, migrate all meetings otherwise only future meetings. - This function is idempotent. It can be run regularly from cron. - """ - if initial: - meetings = Meeting.objects.filter(type='ietf') - MeetingRegistration.objects.filter(reg_type='hackathon').update(reg_type='hackathon_remote') - MeetingRegistration.objects.filter(ticket_type='full_week_pass').update(ticket_type='week_pass') - MeetingRegistration.objects.filter(pk=49645).update(ticket_type='one_day') - MeetingRegistration.objects.filter(pk=50804).update(ticket_type='week_pass') - MeetingRegistration.objects.filter(pk=42386).update(ticket_type='week_pass') - MeetingRegistration.objects.filter(pk=42782).update(ticket_type='one_day') - MeetingRegistration.objects.filter(pk=43464).update(ticket_type='week_pass') - else: - # still process records during week of meeting - one_week_ago = datetime.date.today() - datetime.timedelta(days=7) - meetings = Meeting.objects.filter(type='ietf', date__gt=one_week_ago) - - for meeting in meetings: - # gather all MeetingRegistrations by person (email) - emails = {} - for meeting_reg in MeetingRegistration.objects.filter(meeting=meeting): - if meeting_reg.email in emails: - emails[meeting_reg.email].append(meeting_reg) - else: - emails[meeting_reg.email] = [meeting_reg] - # process each person's registrations - for email, meeting_regs in emails.items(): - preferred_reg = get_preferred(meeting_regs) - reg, created = Registration.objects.get_or_create( - meeting=meeting, - email=email, - defaults={ - 'first_name': preferred_reg.first_name, - 'last_name': preferred_reg.last_name, - 'affiliation': preferred_reg.affiliation, - 'country_code': preferred_reg.country_code, - 'person': preferred_reg.person, - 'attended': preferred_reg.attended, - 'checkedin': preferred_reg.checkedin, - } - ) - if created: - for meeting_reg in meeting_regs: - reg.tickets.create( - attendance_type_id=meeting_reg.reg_type or 'unknown', - ticket_type_id=meeting_reg.ticket_type or 'unknown', - ) - else: - # check if tickets differ - reg_tuple_list = [(t.attendance_type_id, t.ticket_type_id) for t in reg.tickets.all()] - meeting_reg_tuple_list = [(mr.reg_type or 'unknown', mr.ticket_type or 'unknown') for mr in meeting_regs] - if not set(reg_tuple_list) == set(meeting_reg_tuple_list): - # update tickets - reg.tickets.all().delete() - for meeting_reg in meeting_regs: - reg.tickets.create( - attendance_type_id=meeting_reg.reg_type or 'unknown', - ticket_type_id=meeting_reg.ticket_type or 'unknown', - ) - # check fields for updates - fields_to_check = [ - 'first_name', 'last_name', 'affiliation', 'country_code', - 'attended', 'checkedin' - ] - - changed = False - for field in fields_to_check: - new_value = getattr(preferred_reg, field) - if getattr(reg, field) != new_value: - setattr(reg, field, new_value) - changed = True - - if changed: - reg.save() - # delete cancelled Registrations - meeting_reg_email_set = set(emails.keys()) - reg_email_set = set(Registration.objects.filter(meeting=meeting).values_list('email', flat=True)) - for email in reg_email_set - meeting_reg_email_set: - Registration.objects.filter(meeting=meeting, email=email).delete() - - return - - -def check_migrate_registrations(): - """A simple utility function to test that all MeetingRegistration - records got migrated - """ - for mr in MeetingRegistration.objects.all(): - reg = Registration.objects.get(meeting=mr.meeting, email=mr.email) - assert reg.tickets.filter( - attendance_type__slug=mr.reg_type or 'unknown', - ticket_type__slug=mr.ticket_type or 'unknown').exists() - - def generate_proceedings_content(meeting, force_refresh=False): """Render proceedings content for a meeting and update cache diff --git a/ietf/settings.py b/ietf/settings.py index 5e33673611..689f0880a9 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1141,7 +1141,6 @@ def skip_unreadable_post(record): "--outdir" ] -STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://registration.ietf.org/{number}/attendees/' REGISTRATION_PARTICIPANTS_API_URL = 'https://registration.ietf.org/api/v1/participants-dt/' REGISTRATION_PARTICIPANTS_API_KEY = 'changeme' diff --git a/ietf/stats/management/commands/fetch_meeting_attendance.py b/ietf/stats/management/commands/fetch_meeting_attendance.py deleted file mode 100644 index e17ae567fa..0000000000 --- a/ietf/stats/management/commands/fetch_meeting_attendance.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright The IETF Trust 2017-2019, All Rights Reserved -# Copyright 2016 IETF Trust - -from django.core.management.base import BaseCommand, CommandError -from django.utils import timezone - -import debug # pyflakes:ignore - -from ietf.meeting.models import Meeting -from ietf.stats.utils import fetch_attendance_from_meetings -from ietf.utils import log - - -class Command(BaseCommand): - help = "Fetch meeting attendee figures from ietf.org/registration/attendees." - - def add_arguments(self, parser): - parser.add_argument("--meeting", help="meeting to fetch data for") - parser.add_argument("--all", action="store_true", help="fetch data for all meetings") - parser.add_argument("--latest", type=int, help="fetch data for latest N meetings") - - def handle(self, *args, **options): - self.verbosity = options['verbosity'] - - meetings = Meeting.objects.none() - if options['meeting']: - meetings = Meeting.objects.filter(number=options['meeting'], type="ietf") - elif options['all']: - meetings = Meeting.objects.filter(type="ietf").order_by("date") - elif options['latest']: - meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by("-date")[:options['latest']] - else: - raise CommandError("Please use one of --meeting, --all or --latest") - - for meeting, stats in zip(meetings, fetch_attendance_from_meetings(meetings)): - msg = "Fetched data for meeting {:>3}: {:4d} processed, {:4d} added, {:4d} in table".format( - meeting.number, stats.processed, stats.added, stats.total - ) - if self.stdout.isatty(): - self.stdout.write(msg+'\n') # make debugging a bit easier - else: - log.log(msg) diff --git a/ietf/stats/tasks.py b/ietf/stats/tasks.py deleted file mode 100644 index 808e797a40..0000000000 --- a/ietf/stats/tasks.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright The IETF Trust 2024, All Rights Reserved -# -# Celery task definitions -# -from celery import shared_task -from django.utils import timezone - -from ietf.meeting.models import Meeting -from ietf.stats.utils import fetch_attendance_from_meetings -from ietf.utils import log - - -@shared_task -def fetch_meeting_attendance_task(): - # fetch most recent two meetings - meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by("-date")[:2] - try: - stats = fetch_attendance_from_meetings(meetings) - except RuntimeError as err: - log.log(f"Error in fetch_meeting_attendance_task: {err}") - else: - for meeting, meeting_stats in zip(meetings, stats): - log.log( - "Fetched data for meeting {:>3}: {:4d} processed, {:4d} added, {:4d} in table".format( - meeting.number, meeting_stats.processed, meeting_stats.added, meeting_stats.total - ) - ) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 47027277be..48552c8fba 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -3,12 +3,9 @@ import calendar -import datetime import json -from mock import patch from pyquery import PyQuery -from requests import Response import debug # pyflakes:ignore @@ -19,12 +16,8 @@ from ietf.group.factories import RoleFactory -from ietf.meeting.factories import MeetingFactory from ietf.person.factories import PersonFactory from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory -from ietf.stats.models import MeetingRegistration -from ietf.stats.tasks import fetch_meeting_attendance_task -from ietf.stats.utils import get_meeting_registration_data, FetchStats, fetch_attendance_from_meetings from ietf.utils.timezone import date_today @@ -116,120 +109,3 @@ def test_review_stats(self): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('.review-stats td:contains("1")')) - - @patch('requests.get') - def test_get_meeting_registration_data(self, mock_get): - '''Test function to get reg data. Confirm leading/trailing spaces stripped''' - person = PersonFactory() - data = { - 'LastName': person.last_name() + ' ', - 'FirstName': person.first_name(), - 'Company': 'ABC', - 'Country': 'US', - 'Email': person.email().address, - 'RegType': 'onsite', - 'TicketType': 'week_pass', - 'CheckedIn': 'True', - } - data2 = data.copy() - data2['RegType'] = 'hackathon' - response_a = Response() - response_a.status_code = 200 - response_a._content = json.dumps([data, data2]).encode('utf8') - # second response one less record, it's been deleted - response_b = Response() - response_b.status_code = 200 - response_b._content = json.dumps([data]).encode('utf8') - # mock_get.return_value = response - mock_get.side_effect = [response_a, response_b] - meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016, 7, 14), number="96") - get_meeting_registration_data(meeting) - query = MeetingRegistration.objects.filter( - first_name=person.first_name(), - last_name=person.last_name(), - country_code='US') - self.assertEqual(query.count(), 2) - self.assertEqual(query.filter(reg_type='onsite').count(), 1) - self.assertEqual(query.filter(reg_type='hackathon').count(), 1) - onsite = query.get(reg_type='onsite') - self.assertEqual(onsite.ticket_type, 'week_pass') - self.assertEqual(onsite.checkedin, True) - # call a second time to test delete - get_meeting_registration_data(meeting) - query = MeetingRegistration.objects.filter(meeting=meeting, email=person.email()) - self.assertEqual(query.count(), 1) - self.assertEqual(query.filter(reg_type='onsite').count(), 1) - self.assertEqual(query.filter(reg_type='hackathon').count(), 0) - - @patch('requests.get') - def test_get_meeting_registration_data_duplicates(self, mock_get): - '''Test that get_meeting_registration_data does not create duplicate - MeetingRegistration records - ''' - person = PersonFactory() - data = { - 'LastName': person.last_name() + ' ', - 'FirstName': person.first_name(), - 'Company': 'ABC', - 'Country': 'US', - 'Email': person.email().address, - 'RegType': 'onsite', - 'TicketType': 'week_pass', - 'CheckedIn': 'True', - } - data2 = data.copy() - data2['RegType'] = 'hackathon' - response = Response() - response.status_code = 200 - response._content = json.dumps([data, data2, data]).encode('utf8') - mock_get.return_value = response - meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016, 7, 14), number="96") - self.assertEqual(MeetingRegistration.objects.count(), 0) - get_meeting_registration_data(meeting) - query = MeetingRegistration.objects.all() - self.assertEqual(query.count(), 2) - - @patch("ietf.stats.utils.get_meeting_registration_data") - def test_fetch_attendance_from_meetings(self, mock_get_mtg_reg_data): - mock_meetings = [object(), object(), object()] - mock_get_mtg_reg_data.side_effect = ( - (1, 2, 3), - (4, 5, 6), - (7, 8, 9), - ) - stats = fetch_attendance_from_meetings(mock_meetings) - self.assertEqual( - [mock_get_mtg_reg_data.call_args_list[n][0][0] for n in range(3)], - mock_meetings, - ) - self.assertEqual( - stats, - [ - FetchStats(1, 2, 3), - FetchStats(4, 5, 6), - FetchStats(7, 8, 9), - ] - ) - - -class TaskTests(TestCase): - @patch("ietf.stats.tasks.fetch_attendance_from_meetings") - def test_fetch_meeting_attendance_task(self, mock_fetch_attendance): - today = date_today() - meetings = [ - MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=1)), - MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=2)), - MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=3)), - ] - mock_fetch_attendance.return_value = [FetchStats(1,2,3), FetchStats(1,2,3)] - - fetch_meeting_attendance_task() - self.assertEqual(mock_fetch_attendance.call_count, 1) - self.assertCountEqual(mock_fetch_attendance.call_args[0][0], meetings[0:2]) - - # test handling of RuntimeError - mock_fetch_attendance.reset_mock() - mock_fetch_attendance.side_effect = RuntimeError - fetch_meeting_attendance_task() - self.assertTrue(mock_fetch_attendance.called) - # Good enough that we got here without raising an exception diff --git a/ietf/stats/utils.py b/ietf/stats/utils.py index f2e1d9801d..a13e87a4f4 100644 --- a/ietf/stats/utils.py +++ b/ietf/stats/utils.py @@ -3,18 +3,12 @@ import re -import requests -from collections import defaultdict, namedtuple - -from django.conf import settings -from django.db.models import Q +from collections import defaultdict import debug # pyflakes:ignore -from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias, MeetingRegistration +from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias from ietf.name.models import CountryName -from ietf.person.models import Email -from ietf.utils.log import log import logging logger = logging.getLogger('django') @@ -221,120 +215,3 @@ def compute_hirsch_index(citation_counts): i += 1 return i - - -def get_meeting_registration_data(meeting): - """"Retrieve registration attendee data and summary statistics. Returns number - of Registration records created. - - MeetingRegistration records are created in realtime as people register for a - meeting. This function serves as an audit / reconciliation. Most records are - expected to already exist. The function has been optimized with this in mind. - """ - num_created = 0 - num_processed = 0 - try: - response = requests.get( - settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number), - timeout=settings.DEFAULT_REQUESTS_TIMEOUT, - ) - except requests.Timeout as exc: - log(f'GET request timed out for [{settings.STATS_REGISTRATION_ATTENDEES_JSON_URL}]: {exc}') - raise RuntimeError("Timeout retrieving data from registrations API") from exc - if response.status_code == 200: - decoded = [] - try: - decoded = response.json() - except ValueError: - if response.content.strip() == 'Invalid meeting': - logger.info('Invalid meeting: {}'.format(meeting.number)) - return (0,0,0) - else: - raise RuntimeError("Could not decode response from registrations API: '%s...'" % (response.content[:64], )) - - records = MeetingRegistration.objects.filter(meeting_id=meeting.pk).select_related('person') - meeting_registrations = {(r.email, r.reg_type):r for r in records} - for registration in decoded: - person = None - # capture the stripped registration values for later use - first_name = registration['FirstName'].strip() - last_name = registration['LastName'].strip() - affiliation = registration['Company'].strip() - country_code = registration['Country'].strip() - address = registration['Email'].strip() - reg_type = registration['RegType'].strip() - ticket_type = registration['TicketType'].strip() - checkedin = bool(registration['CheckedIn'].strip().lower() == 'true') - - if (address, reg_type) in meeting_registrations: - object = meeting_registrations.pop((address, reg_type)) - created = False - else: - object, created = MeetingRegistration.objects.get_or_create( - meeting_id=meeting.pk, - email=address, - reg_type=reg_type) - - if (object.first_name != first_name[:200] or - object.last_name != last_name[:200] or - object.affiliation != affiliation or - object.country_code != country_code or - object.ticket_type != ticket_type or - object.checkedin != checkedin): - object.first_name=first_name[:200] - object.last_name=last_name[:200] - object.affiliation=affiliation - object.country_code=country_code - object.ticket_type=ticket_type - object.checkedin=checkedin - object.save() - - # Add a Person object to MeetingRegistration object - # if valid email is available - if object and not object.person and address: - # If the person already exists do not try to create a new one - emails = Email.objects.filter(address=address) - # there can only be on Email object with a unique email address (primary key) - if emails.exists(): - person = emails.first().person - # Create a new Person object - else: - logger.error("No Person record for registration. email={}".format(address)) - # update the person object to an actual value - object.person = person - object.save() - - if created: - num_created += 1 - num_processed += 1 - - # any registrations left in meeting_registrations no longer exist in reg - # so must have been deleted - for r in meeting_registrations: - try: - MeetingRegistration.objects.get(meeting=meeting,email=r[0],reg_type=r[1]).delete() - logger.info('Removing deleted registration. email={}, reg_type={}'.format(r[0], r[1])) - except MeetingRegistration.DoesNotExist: - pass - else: - raise RuntimeError("Bad response from registrations API: %s, '%s'" % (response.status_code, response.content)) - num_total = MeetingRegistration.objects.filter( - meeting_id=meeting.pk, - reg_type__in=['onsite', 'remote'] - ).filter( - Q(attended=True) | Q(checkedin=True) - ).count() - if meeting.attendees is None or num_total > meeting.attendees: - meeting.attendees = num_total - meeting.save() - return num_created, num_processed, num_total - - -FetchStats = namedtuple("FetchStats", "added processed total") - - -def fetch_attendance_from_meetings(meetings): - stats = [ - FetchStats(*get_meeting_registration_data(meeting)) for meeting in meetings - ] - return stats