From 2b7d4ad414e61b001cab30703a270929852a6b57 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Fri, 4 Apr 2025 02:19:43 -0400 Subject: [PATCH 01/17] ci: add update staging DB step to deploy workflow --- .github/workflows/build.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 123bd5c65a2..36439aa6cfd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -444,6 +444,19 @@ jobs: PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} steps: + - name: Refresh Staging DB + uses: the-actions-org/workflow-dispatch@v4 + with: + workflow: update-staging-db.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "sourceDb":"datatracker" }' + wait-for-completion: true + wait-for-completion-timeout: 5m + wait-for-completion-interval: 20s + display-workflow-run-url: false + - name: Deploy to staging uses: the-actions-org/workflow-dispatch@v4 with: From f9ca03086493a00ca188cb4cd949ba1831fd452a Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Tue, 8 Apr 2025 20:24:00 -0400 Subject: [PATCH 02/17] fix: unexpected chatlog links (#8774) * fix: unexpected chatlog links * fix: separate handling of polls and chatlogs --- ietf/meeting/tests_views.py | 38 +++++++++++++++++++ ietf/meeting/views.py | 8 +++- .../meeting/session_buttons_include.html | 11 +++--- 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index a93a26b9815..dfb414b61b0 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -7736,6 +7736,44 @@ def test_add_session_recordings(self): self.assertEqual(r.status_code, 404) self.assertFalse(mock_delete.called) + def test_show_chatlog_links(self): + meeting = MeetingFactory(type_id='ietf', number='122') + session = SessionFactory(meeting=meeting) + doc_name = 'chatlog-72-mars-197001010000' + SessionPresentation.objects.create(session=session,document=DocumentFactory(type_id='chatlog', name=doc_name)) + + session_url = urlreverse('ietf.meeting.views.session_details', + kwargs={'num':meeting.number, 'acronym':session.group.acronym}) + + r = self.client.get(session_url) + + self.assertEqual(r.status_code, 200) + + q = PyQuery(r.content) + + # Find the chatlog link in the desktop view + link_chatlog_box = q(f'a[title="Chat logs for {session.group.acronym}"]') + self.assertTrue(link_chatlog_box, 'Expected element with title "Chat logs for {group.acronym}" not found.') + self.assertEqual(link_chatlog_box.attr('href'), '/doc/'+ doc_name) + + # Find the chatlog link in the mobile view + link_chatlog_list = q('li:contains("Chat logs")') + self.assertTrue(link_chatlog_list, 'Expected
  • element containing "Chat logs" not found.') + self.assertEqual(link_chatlog_list.find('a').attr('href'), '/doc/'+ doc_name) + + def test_hide_chatlog_links(self): + # mock meeting and session, but no chatlog document + meeting = MeetingFactory(type_id='ietf', number='122') + session = SessionFactory(meeting=meeting) + + session_url = urlreverse('ietf.meeting.views.session_details', + kwargs={'num':meeting.number, 'acronym':session.group.acronym}) + + r = self.client.get(session_url) + + self.assertEqual(r.status_code, 200) + # validate no links for chat logs exist + self.assertNotContains(r, 'Chat logs') class HasMeetingsTests(TestCase): diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 722bf829e1b..591b934b58e 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -23,6 +23,7 @@ from urllib.parse import parse_qs, unquote, urlencode, urlsplit, urlunsplit, urlparse from tempfile import mkstemp from wsgiref.handlers import format_date_time +from itertools import chain from django import forms from django.core.cache import caches @@ -2496,7 +2497,12 @@ def session_details(request, num, acronym): session.filtered_artifacts.sort(key=lambda d:artifact_types.index(d.document.type.slug)) session.filtered_slides = session.presentations.filter(document__type__slug='slides').order_by('order') session.filtered_drafts = session.presentations.filter(document__type__slug='draft') - session.filtered_chatlog_and_polls = session.presentations.filter(document__type__slug__in=('chatlog', 'polls')).order_by('document__type__slug') + + filtered_polls = session.presentations.filter(document__type__slug=('polls')) + filtered_chatlogs = session.presentations.filter(document__type__slug=('chatlog')) + session.filtered_chatlog_and_polls = chain(filtered_chatlogs, filtered_polls) + session.chatlog = filtered_chatlogs.first() + # TODO FIXME Deleted materials shouldn't be in the presentations for qs in [session.filtered_artifacts,session.filtered_slides,session.filtered_drafts]: qs = [p for p in qs if p.document.get_state_slug(p.document.type_id)!='deleted'] diff --git a/ietf/templates/meeting/session_buttons_include.html b/ietf/templates/meeting/session_buttons_include.html index bb5144b45d6..b1fd92ba5be 100644 --- a/ietf/templates/meeting/session_buttons_include.html +++ b/ietf/templates/meeting/session_buttons_include.html @@ -2,7 +2,8 @@ {% load origin %} {% load static %} {% load textfilters tz %} -{% load ietf_filters %} +{% load ietf_filters session_filters %} + {% origin %} {% if item and item|should_show_agenda_session_buttons %} {% with slug=item.slug session=item.session timeslot=item.timeslot %} @@ -126,10 +127,10 @@ {% else %} {# chat logs #} - {% if meeting.has_chat_logs %} + {% if meeting.has_chat_logs and session.chatlog %} @@ -303,10 +304,10 @@
    • {% else %} {# chat logs #} - {% if meeting.has_chat_logs %} + {% if meeting.has_chat_logs and session.chatlog %}
  • + href="/doc/{{ session.chatlog.document.name }}"> Chat logs
    • From a728cf2440ab5f4acc88714023d3b2e2f834018b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 8 Apr 2025 21:24:22 -0300 Subject: [PATCH 03/17] test: fix apparent typo / tautological test (#8773) Co-authored-by: Robert Sparks --- ietf/api/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/api/tests.py b/ietf/api/tests.py index ac0b37a608f..7cc56c2d0a2 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -1440,7 +1440,7 @@ def test_api_top_level(self): resource_list = r.json() for name in self.apps: - if not name in self.apps: + if not name in resource_list: sys.stderr.write("Expected a REST API resource for %s, but didn't find one\n" % name) for name in self.apps: From 0348dbd865b99692235c2f6e5a75a8064f8113a1 Mon Sep 17 00:00:00 2001 From: Ryan Cross Date: Tue, 15 Apr 2025 07:37:40 -0700 Subject: [PATCH 04/17] feat: new meeting registration implementation (#8408) * feat: new meeting registration implementation * fix: use on_delete=models.PROTECT for certain FKs * fix: convert outlier reg_types,ticket_types on initial migration * fix: fix initial migration mistake * fix: remove management command. add task * fix: move migration 0010 to 0011 to resolve conflict * fix: add missing migration for model changes * fix: add reg type names for unknown * fix: change migration to use 'unknown'. Add test function * fix: merge migrations * fix: rename test_migrate_registrations to check_migrate_registrations * fix: update names.json * fix: fix migration issue and add task * fix: fix broken migration dependency --- ietf/api/tests.py | 193 +++++++++++++++++- ietf/api/urls.py | 1 + ietf/api/views.py | 143 ++++++++++++- ietf/meeting/admin.py | 32 ++- ietf/meeting/factories.py | 28 ++- .../0011_registration_registrationticket.py | 90 ++++++++ ietf/meeting/models.py | 41 +++- ietf/meeting/resources.py | 43 +++- ietf/meeting/tasks.py | 17 ++ ietf/meeting/tests_utils.py | 91 +++++++++ ietf/meeting/utils.py | 109 +++++++++- ietf/name/admin.py | 4 + ietf/name/fixtures/names.json | 140 +++++++++++++ ...ancetypename_registrationtickettypename.py | 47 +++++ .../migrations/0017_populate_new_reg_names.py | 39 ++++ ietf/name/models.py | 5 +- ietf/name/resources.py | 32 ++- 17 files changed, 1045 insertions(+), 10 deletions(-) create mode 100644 ietf/meeting/migrations/0011_registration_registrationticket.py create mode 100644 ietf/meeting/tests_utils.py create mode 100644 ietf/name/migrations/0016_attendancetypename_registrationtickettypename.py create mode 100644 ietf/name/migrations/0017_populate_new_reg_names.py diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 7cc56c2d0a2..23e9ff01038 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2015-2024, All Rights Reserved # -*- coding: utf-8 -*- import base64 +import copy import datetime import json import html @@ -31,7 +32,7 @@ from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory from ietf.group.factories import RoleFactory from ietf.meeting.factories import MeetingFactory, SessionFactory -from ietf.meeting.models import Session +from ietf.meeting.models import Session, Registration from ietf.nomcom.models import Volunteer from ietf.nomcom.factories import NomComFactory, nomcom_kwargs_for_year from ietf.person.factories import PersonFactory, random_faker, EmailFactory, PersonalApiKeyFactory @@ -828,6 +829,196 @@ def test_api_new_meeting_registration_nomcom_volunteer(self): self.assertEqual(volunteer.nomcom, nomcom) self.assertEqual(volunteer.origin, 'registration') + @override_settings(APP_API_TOKENS={"ietf.api.views.api_new_meeting_registration_v2": ["valid-token"]}) + def test_api_new_meeting_registration_v2(self): + meeting = MeetingFactory(type_id='ietf') + person = PersonFactory() + regs = [ + { + 'affiliation': "Alguma Corporação", + 'country_code': 'PT', + 'email': person.email().address, + 'first_name': person.first_name(), + 'last_name': person.last_name(), + 'meeting': str(meeting.number), + 'reg_type': 'onsite', + 'ticket_type': 'week_pass', + 'checkedin': False, + 'is_nomcom_volunteer': False, + 'cancelled': False, + } + ] + + url = urlreverse('ietf.api.views.api_new_meeting_registration_v2') + # + # Test invalid key + r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "invalid-token"}) + self.assertEqual(r.status_code, 403) + # + # Test invalid data + bad_regs = copy.deepcopy(regs) + del(bad_regs[0]['email']) + r = self.client.post(url, data=json.dumps(bad_regs), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 400) + # + # Test valid POST + r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) + # + # Check record + reg = regs[0] + objects = Registration.objects.filter(email=reg['email'], meeting__number=reg['meeting']) + self.assertEqual(objects.count(), 1) + obj = objects[0] + for key in ['affiliation', 'country_code', 'first_name', 'last_name', 'checkedin']: + self.assertEqual(getattr(obj, key), False if key=='checkedin' else reg.get(key) , "Bad data for field '%s'" % key) + self.assertEqual(obj.tickets.count(), 1) + ticket = obj.tickets.first() + self.assertEqual(ticket.ticket_type.slug, regs[0]['ticket_type']) + self.assertEqual(ticket.attendance_type.slug, regs[0]['reg_type']) + self.assertEqual(obj.person, person) + # + # Test update (switch to remote) + regs = [ + { + 'affiliation': "Alguma Corporação", + 'country_code': 'PT', + 'email': person.email().address, + 'first_name': person.first_name(), + 'last_name': person.last_name(), + 'meeting': str(meeting.number), + 'reg_type': 'remote', + 'ticket_type': 'week_pass', + 'checkedin': False, + 'is_nomcom_volunteer': False, + 'cancelled': False, + } + ] + r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) + objects = Registration.objects.filter(email=reg['email'], meeting__number=reg['meeting']) + self.assertEqual(objects.count(), 1) + obj = objects[0] + self.assertEqual(obj.tickets.count(), 1) + ticket = obj.tickets.first() + self.assertEqual(ticket.ticket_type.slug, regs[0]['ticket_type']) + self.assertEqual(ticket.attendance_type.slug, regs[0]['reg_type']) + # + # Test multiple + regs = [ + { + 'affiliation': "Alguma Corporação", + 'country_code': 'PT', + 'email': person.email().address, + 'first_name': person.first_name(), + 'last_name': person.last_name(), + 'meeting': str(meeting.number), + 'reg_type': 'onsite', + 'ticket_type': 'one_day', + 'checkedin': False, + 'is_nomcom_volunteer': False, + 'cancelled': False, + }, + + { + 'affiliation': "Alguma Corporação", + 'country_code': 'PT', + 'email': person.email().address, + 'first_name': person.first_name(), + 'last_name': person.last_name(), + 'meeting': str(meeting.number), + 'reg_type': 'remote', + 'ticket_type': 'week_pass', + 'checkedin': False, + 'is_nomcom_volunteer': False, + 'cancelled': False, + } + ] + + r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) + objects = Registration.objects.filter(email=reg['email'], meeting__number=reg['meeting']) + self.assertEqual(objects.count(), 1) + obj = objects[0] + self.assertEqual(obj.tickets.count(), 2) + self.assertEqual(obj.tickets.filter(attendance_type__slug='onsite').count(), 1) + self.assertEqual(obj.tickets.filter(attendance_type__slug='remote').count(), 1) + + @override_settings(APP_API_TOKENS={"ietf.api.views.api_new_meeting_registration_v2": ["valid-token"]}) + def test_api_new_meeting_registration_v2_cancelled(self): + meeting = MeetingFactory(type_id='ietf') + person = PersonFactory() + regs = [ + { + 'affiliation': "Acme", + 'country_code': 'US', + 'email': person.email().address, + 'first_name': person.first_name(), + 'last_name': person.last_name(), + 'meeting': str(meeting.number), + 'reg_type': 'onsite', + 'ticket_type': 'week_pass', + 'checkedin': False, + 'is_nomcom_volunteer': False, + 'cancelled': False, + } + ] + url = urlreverse('ietf.api.views.api_new_meeting_registration_v2') + self.assertEqual(Registration.objects.count(), 0) + r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) + self.assertEqual(Registration.objects.count(), 1) + regs[0]['cancelled'] = True + r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) + self.assertEqual(Registration.objects.count(), 0) + + @override_settings(APP_API_TOKENS={"ietf.api.views.api_new_meeting_registration_v2": ["valid-token"]}) + def test_api_new_meeting_registration_v2_nomcom(self): + meeting = MeetingFactory(type_id='ietf') + person = PersonFactory() + regs = [ + { + 'affiliation': "Acme", + 'country_code': 'US', + 'email': person.email().address, + 'first_name': person.first_name(), + 'last_name': person.last_name(), + 'meeting': str(meeting.number), + 'reg_type': 'onsite', + 'ticket_type': 'week_pass', + 'checkedin': False, + 'is_nomcom_volunteer': False, + 'cancelled': False, + } + ] + + url = urlreverse('ietf.api.views.api_new_meeting_registration_v2') + now = datetime.datetime.now() + if now.month > 10: + year = now.year + 1 + else: + year = now.year + # create appropriate group and nomcom objects + nomcom = NomComFactory.create(is_accepting_volunteers=True, **nomcom_kwargs_for_year(year)) + + # first test is_nomcom_volunteer False + r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) + # assert no Volunteers exists + self.assertEqual(Volunteer.objects.count(), 0) + + # test is_nomcom_volunteer True + regs[0]['is_nomcom_volunteer'] = True + r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) + # assert Volunteer exists + self.assertEqual(Volunteer.objects.count(), 1) + volunteer = Volunteer.objects.last() + self.assertEqual(volunteer.person, person) + self.assertEqual(volunteer.nomcom, nomcom) + self.assertEqual(volunteer.origin, 'registration') + def test_api_version(self): DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=datetime.timezone.utc), host='testapi.example.com',tz='UTC') url = urlreverse('ietf.api.views.version') diff --git a/ietf/api/urls.py b/ietf/api/urls.py index b0dbaf91ce7..2fcaba60942 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -66,6 +66,7 @@ # Let MeetEcho upload session polls url(r'^notify/session/polls/?$', meeting_views.api_upload_polls), # Let the registration system notify us about registrations + url(r'^notify/meeting/registration/v2/?', api_views.api_new_meeting_registration_v2), url(r'^notify/meeting/registration/?', api_views.api_new_meeting_registration), # OpenID authentication provider url(r'^openid/$', TemplateView.as_view(template_name='api/openid-issuer.html'), name='ietf.api.urls.oidc_issuer'), diff --git a/ietf/api/views.py b/ietf/api/views.py index 2fd9d2730fa..4cc35606274 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -45,7 +45,7 @@ from ietf.ietfauth.utils import role_required from ietf.ietfauth.views import send_account_creation_email from ietf.ipr.utils import ingest_response_email as ipr_ingest_response_email -from ietf.meeting.models import Meeting +from ietf.meeting.models import Meeting, Registration from ietf.nomcom.models import Volunteer, NomCom from ietf.nomcom.utils import ingest_feedback_email as nomcom_ingest_feedback_email from ietf.person.models import Person, Email @@ -236,6 +236,147 @@ def err(code, text): return HttpResponse(status=405) +_new_registration_json_validator = jsonschema.Draft202012Validator( + schema={ + "type": "array", + "items": { + "type": "object", + "properties": { + "meeting": {"type": "string"}, + "first_name": {"type": "string"}, + "last_name": {"type": "string"}, + "affiliation": {"type": "string"}, + "country_code": {"type": "string"}, + "email": {"type": "string"}, + "reg_type": {"type": "string"}, + "ticket_type": {"type": "string"}, + "checkedin": {"type": "boolean"}, + "is_nomcom_volunteer": {"type": "boolean"}, + "cancelled": {"type": "boolean"}, + }, + "required": ["meeting", "first_name", "last_name", "affiliation", "country_code", "email", "reg_type", "ticket_type", "checkedin", "is_nomcom_volunteer", "cancelled"], + "additionalProperties": "false" + } + } +) + + +@requires_api_token +@csrf_exempt +def api_new_meeting_registration_v2(request): + '''REST API to notify the datatracker about a new meeting registration''' + def _http_err(code, text): + return HttpResponse(text, status=code, content_type="text/plain") + + def _api_response(result): + return JsonResponse(data={"result": result}) + + if request.method != "POST": + return _http_err(405, "Method not allowed") + + if request.content_type != "application/json": + return _http_err(415, "Content-Type must be application/json") + + # Validate + try: + payload = json.loads(request.body) + _new_registration_json_validator.validate(payload) + except json.decoder.JSONDecodeError as err: + return _http_err(400, f"JSON parse error at line {err.lineno} col {err.colno}: {err.msg}") + except jsonschema.exceptions.ValidationError as err: + return _http_err(400, f"JSON schema error at {err.json_path}: {err.message}") + except Exception: + return _http_err(400, "Invalid request format") + + # Validate consistency + # - if receive multiple records they should be for same meeting, same person (email) + if len(payload) > 1: + if len(set([r['meeting'] for r in payload])) != 1: + return _http_err(400, "Different meeting values") + if len(set([r['email'] for r in payload])) != 1: + return _http_err(400, "Different email values") + + # Validate meeting + number = payload[0]['meeting'] + try: + meeting = Meeting.objects.get(number=number) + except Meeting.DoesNotExist: + return _http_err(400, "Invalid meeting value: '%s'" % (number, )) + + # Validate email + email = payload[0]['email'] + try: + validate_email(email) + except ValidationError: + return _http_err(400, "Invalid email value: '%s'" % (email, )) + + # get person + person = Person.objects.filter(email__address=email).first() + if not person: + log.log(f"api_new_meeting_registration_v2 no Person found for {email}") + + registration = payload[0] + # handle cancelled + if registration['cancelled']: + if len(payload) > 1: + return _http_err(400, "Error. Received cancelled registration notification with more than one record. ({})".format(email)) + try: + obj = Registration.objects.get(meeting=meeting, email=email) + except Registration.DoesNotExist: + return _http_err(400, "Error. Received cancelled registration notification for non-existing registration. ({})".format(email)) + if obj.tickets.count() == 1: + obj.delete() + else: + obj.tickets.filter( + attendance_type__slug=registration.reg_type, + ticket_type__slug=registration.ticket_type).delete() + return HttpResponse('Success', status=202, content_type='text/plain') + + # create or update MeetingRegistration + update_fields = ['first_name', 'last_name', 'affiliation', 'country_code', 'checkedin', 'is_nomcom_volunteer'] + try: + reg = Registration.objects.get(meeting=meeting, email=email) + for key, value in registration.items(): + if key in update_fields: + setattr(reg, key, value) + reg.save() + except Registration.DoesNotExist: + reg = Registration.objects.create( + meeting_id=meeting.pk, + person=person, + email=email, + first_name=registration['first_name'], + last_name=registration['last_name'], + affiliation=registration['affiliation'], + country_code=registration['country_code'], + checkedin=registration['checkedin']) + + # handle registration tickets + reg.tickets.all().delete() + for registration in payload: + reg.tickets.create( + attendance_type_id=registration['reg_type'], + ticket_type_id=registration['ticket_type'], + ) + # handle nomcom volunteer + if registration['is_nomcom_volunteer'] and person: + try: + nomcom = NomCom.objects.get(is_accepting_volunteers=True) + except (NomCom.DoesNotExist, NomCom.MultipleObjectsReturned): + nomcom = None + if nomcom: + Volunteer.objects.get_or_create( + nomcom=nomcom, + person=person, + defaults={ + "affiliation": registration["affiliation"], + "origin": "registration" + } + ) + + return HttpResponse('Success', status=202, content_type='text/plain') + + def version(request): dumpdate = None dumpinfo = DumpInfo.objects.order_by('-date').first() diff --git a/ietf/meeting/admin.py b/ietf/meeting/admin.py index b7e56c7b77d..5d6adb72945 100644 --- a/ietf/meeting/admin.py +++ b/ietf/meeting/admin.py @@ -7,7 +7,7 @@ from ietf.meeting.models import (Attended, Meeting, Room, Session, TimeSlot, Constraint, Schedule, SchedTimeSessAssignment, ResourceAssociation, FloorPlan, UrlResource, SessionPresentation, ImportantDate, SlideSubmission, SchedulingEvent, BusinessConstraint, - ProceedingsMaterial, MeetingHost) + ProceedingsMaterial, MeetingHost, Registration, RegistrationTicket) class UrlResourceAdmin(admin.ModelAdmin): @@ -213,3 +213,33 @@ class AttendedAdmin(admin.ModelAdmin): search_fields = ["person__name", "session__group__acronym", "session__meeting__number", "session__name", "session__purpose__name"] raw_id_fields= ["person", "session"] admin.site.register(Attended, AttendedAdmin) + +class MeetingFilter(admin.SimpleListFilter): + title = 'Meeting Filter' + parameter_name = 'meeting_id' + + def lookups(self, request, model_admin): + # Your queryset to limit choices + choices = Meeting.objects.filter(type='ietf').values_list('id', 'number') + return choices + + def queryset(self, request, queryset): + if self.value(): + return queryset.filter(meeting__id=self.value()) + return queryset +class RegistrationAdmin(admin.ModelAdmin): + model = Registration + # list_filter = [('meeting', Meeting.objects.filter(type='ietf')), ] + list_filter = [MeetingFilter, ] + list_display = ['meeting', 'first_name', 'last_name', 'affiliation', 'country_code', 'person', 'email', ] + search_fields = ['meeting__number', 'first_name', 'last_name', 'affiliation', 'country_code', 'email', ] + raw_id_fields = ['person'] +admin.site.register(Registration, RegistrationAdmin) + +class RegistrationTicketAdmin(admin.ModelAdmin): + model = RegistrationTicket + list_filter = ['attendance_type', ] + list_display = ['registration', 'attendance_type', 'ticket_type'] + search_fields = ['registration__first_name', 'registration__last_name', 'registration__email'] + raw_id_fields = ['registration'] +admin.site.register(RegistrationTicket, RegistrationTicketAdmin) diff --git a/ietf/meeting/factories.py b/ietf/meeting/factories.py index eb36e9e756d..b3d21830fec 100644 --- a/ietf/meeting/factories.py +++ b/ietf/meeting/factories.py @@ -12,7 +12,7 @@ from ietf.doc.storage_utils import store_str from ietf.meeting.models import (Attended, Meeting, Session, SchedulingEvent, Schedule, TimeSlot, SessionPresentation, FloorPlan, Room, SlideSubmission, Constraint, - MeetingHost, ProceedingsMaterial) + MeetingHost, ProceedingsMaterial, Registration, RegistrationTicket) from ietf.name.models import (ConstraintName, SessionStatusName, ProceedingsMaterialTypeName, TimerangeName, SessionPurposeName) from ietf.doc.factories import ProceedingsMaterialDocFactory @@ -318,3 +318,29 @@ class Meta: session = factory.SubFactory(SessionFactory) person = factory.SubFactory(PersonFactory) + + +class RegistrationFactory(factory.django.DjangoModelFactory): + class Meta: + model = Registration + skip_postgeneration_save = True + + meeting = factory.SubFactory(MeetingFactory) + person = factory.SubFactory(PersonFactory) + email = factory.LazyAttribute(lambda obj: obj.person.email()) + first_name = factory.LazyAttribute(lambda obj: obj.person.first_name()) + last_name = factory.LazyAttribute(lambda obj: obj.person.last_name()) + affiliation = factory.Faker('company') + country_code = factory.Faker('country_code') + attended = False + checkedin = False + + +class RegistrationTicketFactory(factory.django.DjangoModelFactory): + class Meta: + model = RegistrationTicket + skip_postgeneration_save = True + + registration = factory.SubFactory(RegistrationFactory) + attendance_type_id = 'onsite' + ticket_type_id = 'week_pass' diff --git a/ietf/meeting/migrations/0011_registration_registrationticket.py b/ietf/meeting/migrations/0011_registration_registrationticket.py new file mode 100644 index 00000000000..fc161f303d3 --- /dev/null +++ b/ietf/meeting/migrations/0011_registration_registrationticket.py @@ -0,0 +1,90 @@ +# Generated by Django 4.2.20 on 2025-03-16 08:37 + +from django.db import migrations, models +import django.db.models.deletion +import ietf.utils.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("name", "0017_populate_new_reg_names"), + ("person", "0004_alter_person_photo_alter_person_photo_thumb"), + ("meeting", "0010_alter_floorplan_image_alter_meetinghost_logo"), + ] + + operations = [ + migrations.CreateModel( + name="Registration", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("first_name", models.CharField(max_length=255)), + ("last_name", models.CharField(max_length=255)), + ("affiliation", models.CharField(blank=True, max_length=255)), + ("country_code", models.CharField(max_length=2)), + ("email", models.EmailField(blank=True, max_length=254, null=True)), + ("attended", models.BooleanField(default=False)), + ("checkedin", models.BooleanField(default=False)), + ( + "meeting", + ietf.utils.models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="meeting.meeting", + ), + ), + ( + "person", + ietf.utils.models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="person.person", + ), + ), + ], + ), + migrations.CreateModel( + name="RegistrationTicket", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "attendance_type", + ietf.utils.models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + to="name.attendancetypename", + ), + ), + ( + "registration", + ietf.utils.models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="tickets", + to="meeting.registration", + ), + ), + ( + "ticket_type", + ietf.utils.models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + to="name.registrationtickettypename", + ), + ), + ], + ), + ] diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 84d151c3105..003f8cd76e4 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -1,5 +1,5 @@ -# Copyright The IETF Trust 2007-2024, All Rights Reserved # -*- coding: utf-8 -*- +# Copyright The IETF Trust 2007-2024, All Rights Reserved # old meeting models can be found in ../proceedings/models.py @@ -34,7 +34,7 @@ from ietf.name.models import ( MeetingTypeName, TimeSlotTypeName, SessionStatusName, ConstraintName, RoomResourceName, ImportantDateName, TimerangeName, SlideSubmissionStatusName, ProceedingsMaterialTypeName, - SessionPurposeName, + SessionPurposeName, AttendanceTypeName, RegistrationTicketTypeName ) from ietf.person.models import Person from ietf.utils.decorators import memoize @@ -1483,3 +1483,40 @@ class Meta: def __str__(self): return f'{self.person} at {self.session}' + + +class RegistrationManager(models.Manager): + def onsite(self): + return self.get_queryset().filter(registrationticket__attendance_type__slug='onsite') + + def remote(self): + return self.get_queryset().filter(registrationticket__attendance_type__slug='remote').exclude(registrationticket__attendance_type__slug='onsite') + +class Registration(models.Model): + """Registration attendee records from the IETF registration system""" + meeting = ForeignKey(Meeting) + first_name = models.CharField(max_length=255) + last_name = models.CharField(max_length=255) + affiliation = models.CharField(blank=True, max_length=255) + country_code = models.CharField(max_length=2) # ISO 3166 + person = ForeignKey(Person, blank=True, null=True, on_delete=models.PROTECT) + email = models.EmailField(blank=True, null=True) + # attended was used prior to the introduction of the ietf.meeting.Attended model and is still used by + # Meeting.get_attendance() for older meetings. It should not be used except for dealing with legacy data. + attended = models.BooleanField(default=False) + # checkedin indicates that the badge was picked up + checkedin = models.BooleanField(default=False) + + # custom manager + objects = RegistrationManager() + + def __str__(self): + return "{} {}".format(self.first_name, self.last_name) + +class RegistrationTicket(models.Model): + registration = ForeignKey(Registration, related_name='tickets') + attendance_type = ForeignKey(AttendanceTypeName, on_delete=models.PROTECT) + ticket_type = ForeignKey(RegistrationTicketTypeName, on_delete=models.PROTECT) + + def __str__(self): + return "{}:{}".format(self.attendance_type, self.ticket_type) diff --git a/ietf/meeting/resources.py b/ietf/meeting/resources.py index de9ca014765..ede2b5b9939 100644 --- a/ietf/meeting/resources.py +++ b/ietf/meeting/resources.py @@ -14,7 +14,8 @@ from ietf.meeting.models import ( Meeting, ResourceAssociation, Constraint, Room, Schedule, Session, TimeSlot, SchedTimeSessAssignment, SessionPresentation, FloorPlan, UrlResource, ImportantDate, SlideSubmission, SchedulingEvent, - BusinessConstraint, ProceedingsMaterial, MeetingHost, Attended) + BusinessConstraint, ProceedingsMaterial, MeetingHost, Attended, + Registration, RegistrationTicket) from ietf.name.resources import MeetingTypeNameResource class MeetingResource(ModelResource): @@ -431,3 +432,43 @@ class Meta: "session": ALL_WITH_RELATIONS, } api.meeting.register(AttendedResource()) + +from ietf.meeting.resources import MeetingResource +from ietf.person.resources import PersonResource +class RegistrationResource(ModelResource): + meeting = ToOneField(MeetingResource, 'meeting') + person = ToOneField(PersonResource, 'person', null=True) + class Meta: + queryset = Registration.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'registration' + ordering = ['id', ] + filtering = { + "id": ALL, + "first_name": ALL, + "last_name": ALL, + "affiliation": ALL, + "country_code": ALL, + "email": ALL, + "attended": ALL, + "meeting": ALL_WITH_RELATIONS, + "person": ALL_WITH_RELATIONS, + } +api.meeting.register(RegistrationResource()) + +class RegistrationTicketResource(ModelResource): + registration = ToOneField(RegistrationResource, 'registration') + class Meta: + queryset = RegistrationTicket.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'registrationticket' + ordering = ['id', ] + filtering = { + "id": ALL, + "ticket_type": ALL, + "attendance_type": ALL, + "registration": ALL_WITH_RELATIONS, + } +api.meeting.register(RegistrationTicketResource()) diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py index 2b7c2fca91f..f1b7e8e079a 100644 --- a/ietf/meeting/tasks.py +++ b/ietf/meeting/tasks.py @@ -9,6 +9,7 @@ from .models import Meeting from .utils import generate_proceedings_content from .views import generate_agenda_data +from .utils import migrate_registrations, check_migrate_registrations @shared_task @@ -17,6 +18,22 @@ def agenda_data_refresh(): @shared_task +def migrate_registrations_task(initial=False): + """ Migrate ietf.stats.MeetingRegistration to ietf.meeting.Registration + If initial is True, migrate all meetings otherwise only future meetings. + This function is idempotent. It can be run regularly from cron. + """ + migrate_registrations(initial=initial) + + +@shared_task +def check_migrate_registrations_task(): + """ Compare MeetingRegistration with Registration to ensure + all records migrated + """ + check_migrate_registrations() + + def proceedings_content_refresh_task(*, all=False): """Refresh meeting proceedings cache diff --git a/ietf/meeting/tests_utils.py b/ietf/meeting/tests_utils.py new file mode 100644 index 00000000000..4bda3a65fa8 --- /dev/null +++ b/ietf/meeting/tests_utils.py @@ -0,0 +1,91 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +# -*- coding: utf-8 -*- + +import datetime +import debug # pyflakes: ignore +from ietf.meeting.factories import MeetingFactory # RegistrationFactory, RegistrationTicketFactory +from ietf.meeting.models import Registration +from ietf.meeting.utils import migrate_registrations, get_preferred +from ietf.stats.factories import MeetingRegistrationFactory +from ietf.utils.test_utils import TestCase + + +class MigrateRegistrationsTests(TestCase): + def test_new_meeting_registration(self): + meeting = MeetingFactory(type_id='ietf', number='109') + reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass') + self.assertEqual(Registration.objects.count(), 0) + migrate_registrations(initial=True) + self.assertEqual(Registration.objects.count(), 1) + new = Registration.objects.first() + self.assertEqual(new.first_name, reg.first_name) + self.assertEqual(new.last_name, reg.last_name) + self.assertEqual(new.email, reg.email) + self.assertEqual(new.person, reg.person) + self.assertEqual(new.meeting, meeting) + self.assertEqual(new.affiliation, reg.affiliation) + self.assertEqual(new.country_code, reg.country_code) + self.assertEqual(new.checkedin, reg.checkedin) + self.assertEqual(new.attended, reg.attended) + + def test_migrate_non_initial(self): + # with only old meeting + meeting = MeetingFactory(type_id='ietf', number='109') + MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass') + self.assertEqual(Registration.objects.count(), 0) + migrate_registrations() + self.assertEqual(Registration.objects.count(), 0) + # with new meeting + new_meeting = MeetingFactory(type_id='ietf', number='150') + new_meeting.date = datetime.date.today() + datetime.timedelta(days=30) + new_meeting.save() + MeetingRegistrationFactory(meeting=new_meeting, reg_type='onsite', ticket_type='week_pass') + migrate_registrations() + self.assertEqual(Registration.objects.count(), 1) + + def test_updated_meeting_registration(self): + # setup test initial conditions + meeting = MeetingFactory(type_id='ietf', number='109') + reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass') + migrate_registrations(initial=True) + # change first_name and save + original = reg.first_name + reg.first_name = 'NewBob' + reg.save() + new = Registration.objects.first() + self.assertEqual(new.first_name, original) + migrate_registrations(initial=True) + new.refresh_from_db() + self.assertEqual(new.first_name, reg.first_name) + + def test_additional_ticket(self): + # setup test initial conditions + meeting = MeetingFactory(type_id='ietf', number='109') + reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass') + migrate_registrations(initial=True) + new = Registration.objects.first() + self.assertEqual(new.tickets.count(), 1) + # add a second ticket + reg.reg_type = 'remote' + reg.pk = None + reg.save() + migrate_registrations(initial=True) + # new.refresh_from_db() + self.assertEqual(new.tickets.count(), 2) + + def test_cancelled_registration(self): + # setup test initial conditions + meeting = MeetingFactory(type_id='ietf', number='109') + reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass') + migrate_registrations(initial=True) + reg.delete() + # do test + migrate_registrations(initial=True) + self.assertEqual(Registration.objects.count(), 0) + + def test_get_preferred(self): + meeting = MeetingFactory(type_id='ietf', number='109') + onsite = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass') + hackathon = MeetingRegistrationFactory(meeting=meeting, reg_type='hackathon_onsite', ticket_type='week_pass') + result = get_preferred([onsite, hackathon]) + self.assertEqual(result, onsite) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 9603278399f..d35914fd60a 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -26,7 +26,8 @@ from ietf.dbtemplate.models import DBTemplate from ietf.doc.storage_utils import store_bytes, store_str from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot, - Constraint, SchedTimeSessAssignment, SessionPresentation, Attended) + Constraint, SchedTimeSessAssignment, SessionPresentation, Attended, + Registration, Meeting) from ietf.doc.models import Document, State, NewRevisionDocEvent, StateDocEvent from ietf.doc.models import DocEvent from ietf.group.models import Group @@ -1013,6 +1014,112 @@ def participants_for_meeting(meeting): return (checked_in, attended) +def get_preferred(regs): + """ Return a preferred regular registration (non hackathon) from + a list of registrations if there is one, otherwise any. + """ + for reg in regs: + if reg.reg_type in ['onsite', 'remote']: + return reg + return reg + + +def migrate_registrations(initial=False): + """ Migrate ietf.stats.MeetingRegistration to ietf.meeting.Registration + If initial is True, migrate all meetings otherwise only future meetings. + This function is idempotent. It can be run regularly from cron. + """ + if initial: + meetings = Meeting.objects.filter(type='ietf') + MeetingRegistration.objects.filter(reg_type='hackathon').update(reg_type='hackathon_remote') + MeetingRegistration.objects.filter(ticket_type='full_week_pass').update(ticket_type='week_pass') + MeetingRegistration.objects.filter(pk=49645).update(ticket_type='one_day') + MeetingRegistration.objects.filter(pk=50804).update(ticket_type='week_pass') + MeetingRegistration.objects.filter(pk=42386).update(ticket_type='week_pass') + MeetingRegistration.objects.filter(pk=42782).update(ticket_type='one_day') + MeetingRegistration.objects.filter(pk=43464).update(ticket_type='week_pass') + else: + # still process records during week of meeting + one_week_ago = datetime.date.today() - datetime.timedelta(days=7) + meetings = Meeting.objects.filter(type='ietf', date__gt=one_week_ago) + + for meeting in meetings: + # gather all MeetingRegistrations by person (email) + emails = {} + for meeting_reg in MeetingRegistration.objects.filter(meeting=meeting): + if meeting_reg.email in emails: + emails[meeting_reg.email].append(meeting_reg) + else: + emails[meeting_reg.email] = [meeting_reg] + # process each person's registrations + for email, meeting_regs in emails.items(): + preferred_reg = get_preferred(meeting_regs) + reg, created = Registration.objects.get_or_create( + meeting=meeting, + email=email, + defaults={ + 'first_name': preferred_reg.first_name, + 'last_name': preferred_reg.last_name, + 'affiliation': preferred_reg.affiliation, + 'country_code': preferred_reg.country_code, + 'person': preferred_reg.person, + 'attended': preferred_reg.attended, + 'checkedin': preferred_reg.checkedin, + } + ) + if created: + for meeting_reg in meeting_regs: + reg.tickets.create( + attendance_type_id=meeting_reg.reg_type or 'unknown', + ticket_type_id=meeting_reg.ticket_type or 'unknown', + ) + else: + # check if tickets differ + reg_tuple_list = [(t.attendance_type_id, t.ticket_type_id) for t in reg.tickets.all()] + meeting_reg_tuple_list = [(mr.reg_type or 'unknown', mr.ticket_type or 'unknown') for mr in meeting_regs] + if not set(reg_tuple_list) == set(meeting_reg_tuple_list): + # update tickets + reg.tickets.all().delete() + for meeting_reg in meeting_regs: + reg.tickets.create( + attendance_type_id=meeting_reg.reg_type or 'unknown', + ticket_type_id=meeting_reg.ticket_type or 'unknown', + ) + # check fields for updates + fields_to_check = [ + 'first_name', 'last_name', 'affiliation', 'country_code', + 'attended', 'checkedin' + ] + + changed = False + for field in fields_to_check: + new_value = getattr(preferred_reg, field) + if getattr(reg, field) != new_value: + setattr(reg, field, new_value) + changed = True + + if changed: + reg.save() + # delete cancelled Registrations + meeting_reg_email_set = set(emails.keys()) + reg_email_set = set(Registration.objects.filter(meeting=meeting).values_list('email', flat=True)) + for email in reg_email_set - meeting_reg_email_set: + Registration.objects.filter(meeting=meeting, email=email).delete() + + return + + +def check_migrate_registrations(): + """A simple utility function to test that all MeetingRegistration + records got migrated + """ + for mr in MeetingRegistration.objects.all(): + reg = Registration.objects.get(meeting=mr.meeting, email=mr.email) + assert reg.tickets.filter( + attendance_type__slug=mr.reg_type or 'unknown', + ticket_type__slug=mr.ticket_type or 'unknown').exists() + + def generate_proceedings_content(meeting, force_refresh=False): """Render proceedings content for a meeting and update cache diff --git a/ietf/name/admin.py b/ietf/name/admin.py index 2458da37d9b..4336e0569c7 100644 --- a/ietf/name/admin.py +++ b/ietf/name/admin.py @@ -3,6 +3,7 @@ from ietf.name.models import ( AgendaTypeName, + AttendanceTypeName, BallotPositionName, ConstraintName, ContinentName, @@ -29,6 +30,7 @@ LiaisonStatementTagName, MeetingTypeName, NomineePositionStateName, + RegistrationTicketTypeName, ReviewRequestStateName, ReviewResultName, ReviewTypeName, @@ -137,6 +139,7 @@ class ProceedingsMaterialTypeNameAdmin(NameAdmin): admin.site.register(AgendaFilterTypeName, NameAdmin) admin.site.register(AgendaTypeName, NameAdmin) admin.site.register(AppealArtifactTypeName, NameAdmin) +admin.site.register(AttendanceTypeName, NameAdmin) admin.site.register(BallotPositionName, NameAdmin) admin.site.register(ConstraintName, NameAdmin) admin.site.register(ContinentName, NameAdmin) @@ -158,6 +161,7 @@ class ProceedingsMaterialTypeNameAdmin(NameAdmin): admin.site.register(LiaisonStatementTagName, NameAdmin) admin.site.register(MeetingTypeName, NameAdmin) admin.site.register(NomineePositionStateName, NameAdmin) +admin.site.register(RegistrationTicketTypeName, NameAdmin) admin.site.register(ReviewRequestStateName, NameAdmin) admin.site.register(ReviewAssignmentStateName, NameAdmin) admin.site.register(ReviewResultName, NameAdmin) diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 96273fbc35f..15ae71d8497 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -6827,6 +6827,66 @@ "model": "name.appealartifacttypename", "pk": "response" }, + { + "fields": { + "desc": "", + "name": "ANRW Onsite", + "order": 0, + "used": true + }, + "model": "name.attendancetypename", + "pk": "anrw_onsite" + }, + { + "fields": { + "desc": "", + "name": "Hackathon Onsite", + "order": 0, + "used": true + }, + "model": "name.attendancetypename", + "pk": "hackathon_onsite" + }, + { + "fields": { + "desc": "", + "name": "Hackathon Remote", + "order": 0, + "used": true + }, + "model": "name.attendancetypename", + "pk": "hackathon_remote" + }, + { + "fields": { + "desc": "", + "name": "Onsite", + "order": 0, + "used": true + }, + "model": "name.attendancetypename", + "pk": "onsite" + }, + { + "fields": { + "desc": "", + "name": "Remote", + "order": 0, + "used": true + }, + "model": "name.attendancetypename", + "pk": "remote" + }, + { + "fields": { + "desc": "", + "name": "Unknown", + "order": 0, + "used": true + }, + "model": "name.attendancetypename", + "pk": "unknown" + }, { "fields": { "blocking": false, @@ -12877,6 +12937,86 @@ "model": "name.proceedingsmaterialtypename", "pk": "wiki" }, + { + "fields": { + "desc": "", + "name": "ANRW Combo", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "anrw_combo" + }, + { + "fields": { + "desc": "", + "name": "ANRW Only", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "anrw_only" + }, + { + "fields": { + "desc": "", + "name": "Hackathon Combo", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "hackathon_combo" + }, + { + "fields": { + "desc": "", + "name": "Hackathon Only", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "hackathon_only" + }, + { + "fields": { + "desc": "", + "name": "One Day", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "one_day" + }, + { + "fields": { + "desc": "", + "name": "Student", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "student" + }, + { + "fields": { + "desc": "", + "name": "Unknown", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "unknown" + }, + { + "fields": { + "desc": "", + "name": "Week Pass", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "week_pass" + }, { "fields": { "desc": "The reviewer has accepted the assignment", diff --git a/ietf/name/migrations/0016_attendancetypename_registrationtickettypename.py b/ietf/name/migrations/0016_attendancetypename_registrationtickettypename.py new file mode 100644 index 00000000000..9376d3a4c6b --- /dev/null +++ b/ietf/name/migrations/0016_attendancetypename_registrationtickettypename.py @@ -0,0 +1,47 @@ +# Generated by Django 4.2.17 on 2025-01-02 18:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("name", "0015_last_call_name"), + ] + + operations = [ + migrations.CreateModel( + name="AttendanceTypeName", + fields=[ + ( + "slug", + models.CharField(max_length=32, primary_key=True, serialize=False), + ), + ("name", models.CharField(max_length=255)), + ("desc", models.TextField(blank=True)), + ("used", models.BooleanField(default=True)), + ("order", models.IntegerField(default=0)), + ], + options={ + "ordering": ["order", "name"], + "abstract": False, + }, + ), + migrations.CreateModel( + name="RegistrationTicketTypeName", + fields=[ + ( + "slug", + models.CharField(max_length=32, primary_key=True, serialize=False), + ), + ("name", models.CharField(max_length=255)), + ("desc", models.TextField(blank=True)), + ("used", models.BooleanField(default=True)), + ("order", models.IntegerField(default=0)), + ], + options={ + "ordering": ["order", "name"], + "abstract": False, + }, + ), + ] diff --git a/ietf/name/migrations/0017_populate_new_reg_names.py b/ietf/name/migrations/0017_populate_new_reg_names.py new file mode 100644 index 00000000000..51954885c0b --- /dev/null +++ b/ietf/name/migrations/0017_populate_new_reg_names.py @@ -0,0 +1,39 @@ +# Generated by Django 4.2.17 on 2025-01-02 18:26 + +from django.db import migrations + +def forward(apps, schema_editor): + AttendanceTypeName = apps.get_model('name', 'AttendanceTypeName') + RegistrationTicketTypeName = apps.get_model('name', 'RegistrationTicketTypeName') + AttendanceTypeName.objects.create(slug='onsite', name='Onsite') + AttendanceTypeName.objects.create(slug='remote', name='Remote') + AttendanceTypeName.objects.create(slug='hackathon_onsite', name='Hackathon Onsite') + AttendanceTypeName.objects.create(slug='hackathon_remote', name='Hackathon Remote') + AttendanceTypeName.objects.create(slug='anrw_onsite', name='ANRW Onsite') + AttendanceTypeName.objects.create(slug='unknown', name='Unknown') + RegistrationTicketTypeName.objects.create(slug='week_pass', name='Week Pass') + RegistrationTicketTypeName.objects.create(slug='one_day', name='One Day') + RegistrationTicketTypeName.objects.create(slug='student', name='Student') + RegistrationTicketTypeName.objects.create(slug='hackathon_only', name='Hackathon Only') + RegistrationTicketTypeName.objects.create(slug='hackathon_combo', name='Hackathon Combo') + RegistrationTicketTypeName.objects.create(slug='anrw_only', name='ANRW Only') + RegistrationTicketTypeName.objects.create(slug='anrw_combo', name='ANRW Combo') + RegistrationTicketTypeName.objects.create(slug='unknown', name='Unknown') + + +def reverse(apps, schema_editor): + AttendanceTypeName = apps.get_model('name', 'AttendanceTypeName') + RegistrationTicketTypeName = apps.get_model('name', 'RegistrationTicketTypeName') + AttendanceTypeName.objects.delete() + RegistrationTicketTypeName.objects.delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ("name", "0016_attendancetypename_registrationtickettypename"), + ] + + operations = [ + migrations.RunPython(forward, reverse), + ] diff --git a/ietf/name/models.py b/ietf/name/models.py index 8c2520a4896..0e87d43548f 100644 --- a/ietf/name/models.py +++ b/ietf/name/models.py @@ -151,6 +151,9 @@ class SlideSubmissionStatusName(NameModel): "Pending, Accepted, Rejected" class TelechatAgendaSectionName(NameModel): """roll_call, minutes, action_items""" - class AppealArtifactTypeName(NameModel): pass +class AttendanceTypeName(NameModel): + """onsite, remote, hackathon_onsite, hackathon_remote""" +class RegistrationTicketTypeName(NameModel): + """week, one_day, student""" diff --git a/ietf/name/resources.py b/ietf/name/resources.py index dffa7669dba..0cb0e41e0b7 100644 --- a/ietf/name/resources.py +++ b/ietf/name/resources.py @@ -19,7 +19,7 @@ RoleName, RoomResourceName, SessionStatusName, StdLevelName, StreamName, TimeSlotTypeName, TopicAudienceName, ReviewerQueuePolicyName, TimerangeName, ExtResourceTypeName, ExtResourceName, SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName, TelechatAgendaSectionName, - AppealArtifactTypeName ) + AppealArtifactTypeName, AttendanceTypeName, RegistrationTicketTypeName ) class TimeSlotTypeNameResource(ModelResource): class Meta: @@ -752,3 +752,33 @@ class Meta: "order": ALL, } api.name.register(AppealArtifactTypeNameResource()) + + +class AttendanceTypeNameResource(ModelResource): + class Meta: + cache = SimpleCache() + queryset = AttendanceTypeName.objects.all() + serializer = api.Serializer() + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + } +api.name.register(AttendanceTypeNameResource()) + + +class RegistrationTicketTypeNameResource(ModelResource): + class Meta: + cache = SimpleCache() + queryset = RegistrationTicketTypeName.objects.all() + serializer = api.Serializer() + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + } +api.name.register(RegistrationTicketTypeNameResource()) From d5e3840d4dbeec37415fae89ab578b1b71046b6a Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 15 Apr 2025 11:17:33 -0400 Subject: [PATCH 05/17] fix: 500->400 for bad /api/v1/ requests (#8802) * fix: 400->500 for bad /api/v1/ requests * chore: remove unused import --- ietf/api/views.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ietf/api/views.py b/ietf/api/views.py index 4cc35606274..e5b1f96aed8 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -18,7 +18,7 @@ from django.contrib.auth.models import User from django.core.exceptions import ValidationError from django.core.validators import validate_email -from django.http import HttpResponse, Http404, JsonResponse +from django.http import HttpResponse, Http404, JsonResponse, HttpResponseBadRequest from django.shortcuts import render, get_object_or_404 from django.urls import reverse from django.utils.decorators import method_decorator @@ -68,7 +68,10 @@ def top_level(request): } serializer = Serializer() - desired_format = determine_format(request, serializer) + try: + desired_format = determine_format(request, serializer) + except BadRequest as err: + return HttpResponseBadRequest(str(err)) options = {} @@ -76,10 +79,12 @@ def top_level(request): callback = request.GET.get('callback', 'callback') if not is_valid_jsonp_callback_value(callback): - raise BadRequest('JSONP callback name is invalid.') + return HttpResponseBadRequest("JSONP callback name is invalid") options['callback'] = callback + # This might raise UnsupportedFormat, but that indicates a real server misconfiguration + # so let it bubble up unhandled and trigger a 500 / email to admins. serialized = serializer.serialize(available_resources, desired_format, options) return HttpResponse(content=serialized, content_type=build_content_type(desired_format)) From 86988eb6072b1331c393f5f7999350f45d653f04 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Tue, 15 Apr 2025 11:45:48 -0400 Subject: [PATCH 06/17] fix: remove credentials omit from agenda fetch call (#8779) --- client/agenda/store.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/agenda/store.js b/client/agenda/store.js index 359c5fbf055..b5498303a68 100644 --- a/client/agenda/store.js +++ b/client/agenda/store.js @@ -141,7 +141,7 @@ export const useAgendaStore = defineStore('agenda', { meetingNumber = meetingData.meetingNumber } - const resp = await fetch(`/api/meeting/${meetingNumber}/agenda-data`, { credentials: 'omit' }) + const resp = await fetch(`/api/meeting/${meetingNumber}/agenda-data`) if (!resp.ok) { throw new Error(resp.statusText) } From be06d7c4d34a6fc18bc30d4a064dc3ffcf18c2cf Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 16 Apr 2025 12:00:04 -0500 Subject: [PATCH 07/17] chore: reorder migrations (#8804) * chore: reorder migrations * chore: copyright --- ...ationticket.py => 0012_registration_registrationticket.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename ietf/meeting/migrations/{0011_registration_registrationticket.py => 0012_registration_registrationticket.py} (96%) diff --git a/ietf/meeting/migrations/0011_registration_registrationticket.py b/ietf/meeting/migrations/0012_registration_registrationticket.py similarity index 96% rename from ietf/meeting/migrations/0011_registration_registrationticket.py rename to ietf/meeting/migrations/0012_registration_registrationticket.py index fc161f303d3..c555f52e8bb 100644 --- a/ietf/meeting/migrations/0011_registration_registrationticket.py +++ b/ietf/meeting/migrations/0012_registration_registrationticket.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.20 on 2025-03-16 08:37 +# Copyright The IETF Trust 2025, All Rights Reserved from django.db import migrations, models import django.db.models.deletion @@ -10,7 +10,7 @@ class Migration(migrations.Migration): dependencies = [ ("name", "0017_populate_new_reg_names"), ("person", "0004_alter_person_photo_alter_person_photo_thumb"), - ("meeting", "0010_alter_floorplan_image_alter_meetinghost_logo"), + ("meeting", "0011_alter_slidesubmission_doc"), ] operations = [ From 3d69b2c03e24d78f28a824e04f2254ff33f651d7 Mon Sep 17 00:00:00 2001 From: Ryan Cross Date: Wed, 16 Apr 2025 12:46:27 -0700 Subject: [PATCH 08/17] feat: add API for related emails (#8671) * feat: add api for related emails. Fixes #8275 * fix: switch from querystring parameter to URL parameter * fix: exclude null character in URL regex --------- Co-authored-by: Robert Sparks --- ietf/api/tests.py | 33 +++++++++++++++++++++++++++++++++ ietf/api/urls.py | 2 ++ ietf/api/views.py | 25 +++++++++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 23e9ff01038..809b45cc2b3 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -1164,6 +1164,39 @@ def test_active_email_list(self): self.assertCountEqual(result.keys(), ["addresses"]) self.assertCountEqual(result["addresses"], Email.objects.filter(active=True).values_list("address", flat=True)) + @override_settings(APP_API_TOKENS={"ietf.api.views.related_email_list": ["valid-token"]}) + def test_related_email_list(self): + joe = EmailFactory(address='joe@work.com') + EmailFactory(address='joe@home.com', person=joe.person) + EmailFactory(address='jòe@spain.com', person=joe.person) + url = urlreverse("ietf.api.views.related_email_list", kwargs={'email': 'joe@home.com'}) + # no api key + r = self.client.get(url, headers={}) + self.assertEqual(r.status_code, 403) + # invalid api key + r = self.client.get(url, headers={"X-Api-Key": "not-the-valid-token"}) + self.assertEqual(r.status_code, 403) + # wrong method + r = self.client.post(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 405) + # valid + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + result = json.loads(r.content) + self.assertCountEqual(result.keys(), ["addresses"]) + self.assertCountEqual(result["addresses"], joe.person.email_set.exclude(address='joe@home.com').values_list("address", flat=True)) + # non-ascii + non_ascii_url = urlreverse("ietf.api.views.related_email_list", kwargs={'email': 'jòe@spain.com'}) + r = self.client.get(non_ascii_url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + result = json.loads(r.content) + self.assertTrue('joe@home.com' in result["addresses"]) + # email not found + not_found_url = urlreverse("ietf.api.views.related_email_list", kwargs={'email': 'nobody@nowhere.com'}) + r = self.client.get(not_found_url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 404) + @override_settings(APP_API_TOKENS={"ietf.api.views.role_holder_addresses": ["valid-token"]}) def test_role_holder_addresses(self): url = urlreverse("ietf.api.views.role_holder_addresses") diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 2fcaba60942..bafd5c5b761 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -73,6 +73,8 @@ url(r'^openid/', include('oidc_provider.urls', namespace='oidc_provider')), # Email alias listing url(r'^person/email/$', api_views.active_email_list), + # Related Email listing + url(r'^person/email/(?P[^/\x00]+)/related/$', api_views.related_email_list), # Draft submission API url(r'^submit/?$', submit_views.api_submit_tombstone), # Draft upload API diff --git a/ietf/api/views.py b/ietf/api/views.py index e5b1f96aed8..97b97930482 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -691,6 +691,31 @@ def active_email_list(request): return HttpResponse(status=405) +@requires_api_token +@csrf_exempt +def related_email_list(request, email): + """Given an email address, returns all other email addresses known + to Datatracker, via Person object + """ + def _http_err(code, text): + return HttpResponse(text, status=code, content_type="text/plain") + + if request.method == "GET": + try: + email_obj = Email.objects.get(address=email) + except Email.DoesNotExist: + return _http_err(404, "Email not found") + person = email_obj.person + if not person: + return JsonResponse({"addresses": []}) + return JsonResponse( + { + "addresses": list(person.email_set.exclude(address=email).values_list("address", flat=True)), + } + ) + return HttpResponse(status=405) + + @requires_api_token def role_holder_addresses(request): if request.method == "GET": From 4a5716e2e89044981905e9af08775ba2f5386fa7 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Fri, 18 Apr 2025 19:39:32 -0400 Subject: [PATCH 09/17] ci: Update build workflow to handle AWS breaking things for no reason --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36439aa6cfd..e5c627c950b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -235,7 +235,7 @@ jobs: docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh echo "Pushing statics..." cd static - aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors + aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors --checksum-algorithm CRC32 - name: Augment dockerignore for docker image build env: From 24042631348135dd8b345715526815f245f9ad51 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 21 Apr 2025 13:00:32 -0300 Subject: [PATCH 10/17] refactor: smtpd->aiosmtpd (#8805) * refactor: smtpd -> aiosmtpd * test: set mock return value for EmailOnFailureCommandTests The test has been working, but in a broken way, for as long as it has existed. The smtpd-based test_smtpserver was masking an exception that did not interfere with the test's effectiveness. * test: increase SMTP.line_length_limit --- ietf/utils/management/tests.py | 2 +- ietf/utils/test_runner.py | 2 +- ietf/utils/test_smtpserver.py | 116 ++++++++++++--------------------- requirements.txt | 1 + 4 files changed, 43 insertions(+), 78 deletions(-) diff --git a/ietf/utils/management/tests.py b/ietf/utils/management/tests.py index e94c39354f0..d704999cd15 100644 --- a/ietf/utils/management/tests.py +++ b/ietf/utils/management/tests.py @@ -12,7 +12,7 @@ from ietf.utils.test_utils import TestCase -@mock.patch.object(EmailOnFailureCommand, 'handle') +@mock.patch.object(EmailOnFailureCommand, 'handle', return_value=None) class EmailOnFailureCommandTests(TestCase): def test_calls_handle(self, handle_method): call_command(EmailOnFailureCommand()) diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index a77377ffb58..c06e7876db6 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -863,7 +863,7 @@ def setup_test_environment(self, **kwargs): try: # remember the value so ietf.utils.mail.send_smtp() will use the same ietf.utils.mail.SMTP_ADDR['port'] = base + offset - self.smtpd_driver = SMTPTestServerDriver((ietf.utils.mail.SMTP_ADDR['ip4'],ietf.utils.mail.SMTP_ADDR['port']),None) + self.smtpd_driver = SMTPTestServerDriver(ietf.utils.mail.SMTP_ADDR['ip4'],ietf.utils.mail.SMTP_ADDR['port'], None) self.smtpd_driver.start() print((" Running an SMTP test server on %(ip4)s:%(port)s to catch outgoing email." % ietf.utils.mail.SMTP_ADDR)) break diff --git a/ietf/utils/test_smtpserver.py b/ietf/utils/test_smtpserver.py index 66675aa0b15..40da758d668 100644 --- a/ietf/utils/test_smtpserver.py +++ b/ietf/utils/test_smtpserver.py @@ -1,92 +1,56 @@ -# Copyright The IETF Trust 2014-2020, All Rights Reserved +# Copyright The IETF Trust 2014-2025, All Rights Reserved # -*- coding: utf-8 -*- +from aiosmtpd.controller import Controller +from aiosmtpd.smtp import SMTP +from email.utils import parseaddr +from typing import Optional -import smtpd -import threading -import asyncore -import debug # pyflakes:ignore +class SMTPTestHandler: -class AsyncCoreLoopThread(object): + def __init__(self, inbox: list): + self.inbox = inbox - def wrap_loop(self, exit_condition, timeout=1.0, use_poll=False, map=None): - if map is None: - map = asyncore.socket_map - while map and not exit_condition: - asyncore.loop(timeout=1.0, use_poll=False, map=map, count=1) + async def handle_DATA(self, server, session, envelope): + """Handle the DATA command and 'deliver' the message""" - def start(self): - """Start the listening service""" - self.exit_condition = [] - kwargs={'exit_condition':self.exit_condition,'timeout':1.0} - self.thread = threading.Thread(target=self.wrap_loop, kwargs=kwargs) - self.thread.daemon = True - self.thread.daemon = True - self.thread.start() - - def stop(self): - """Stop the listening service""" - self.exit_condition.append(True) - self.thread.join() - - -class SMTPTestChannel(smtpd.SMTPChannel): + self.inbox.append(envelope.content) + # Per RFC2033: https://datatracker.ietf.org/doc/html/rfc2033.html#section-4.2 + # ...after the final ".", the server returns one reply + # for each previously successful RCPT command in the mail transaction, + # in the order that the RCPT commands were issued. Even if there were + # multiple successful RCPT commands giving the same forward-path, there + # must be one reply for each successful RCPT command. + return "\n".join("250 OK" for _ in envelope.rcpt_tos) -# mail_options = ['BODY=8BITMIME', 'SMTPUTF8'] - - def smtp_RCPT(self, arg): - if not self.mailfrom: - self.push(str('503 Error: need MAIL command')) - return - arg = self._strip_command_keyword('TO:', arg) - address, __ = self._getaddr(arg) - if not address: - self.push(str('501 Syntax: RCPT TO:
    ')) - return + async def handle_RCPT(self, server, session, envelope, address, rcpt_options): + """Handle an RCPT command and add the address to the envelope if it is acceptable""" + _, address = parseaddr(address) + if address == "": + return "501 Syntax: RCPT TO:
    " if "poison" in address: - self.push(str('550 Error: Not touching that')) - return - self.rcpt_options = [] - self.rcpttos.append(address) - self.push(str('250 Ok')) - -class SMTPTestServer(smtpd.SMTPServer): - - def __init__(self,localaddr,remoteaddr,inbox): - if inbox is not None: - self.inbox=inbox - else: - self.inbox = [] - smtpd.SMTPServer.__init__(self,localaddr,remoteaddr) + return "550 Error: Not touching that" + # At this point the address is acceptable + envelope.rcpt_tos.append(address) + return "250 OK" - def handle_accept(self): - pair = self.accept() - if pair is not None: - conn, addr = pair - #channel = SMTPTestChannel(self, conn, addr) - SMTPTestChannel(self, conn, addr) - def process_message(self, peer, mailfrom, rcpttos, data, mail_options=None, rcpt_options=None): - self.inbox.append(data) +class SMTPTestServerDriver: - -class SMTPTestServerDriver(object): - def __init__(self, localaddr, remoteaddr, inbox=None): - self.localaddr=localaddr - self.remoteaddr=remoteaddr - if inbox is not None: - self.inbox = inbox - else: - self.inbox = [] - self.thread_driver = None + def __init__(self, address: str, port: int, inbox: Optional[list] = None): + # Allow longer lines than the 1001 that RFC 5321 requires. As of 2025-04-16 the + # datatracker emits some non-compliant messages. + # See https://aiosmtpd.aio-libs.org/en/latest/smtp.html + SMTP.line_length_limit = 4000 # tests start failing between 3000 and 4000 + self.controller = Controller( + hostname=address, + port=port, + handler=SMTPTestHandler(inbox=[] if inbox is None else inbox), + ) def start(self): - self.smtpserver = SMTPTestServer(self.localaddr,self.remoteaddr,self.inbox) - self.thread_driver = AsyncCoreLoopThread() - self.thread_driver.start() + self.controller.start() def stop(self): - if self.thread_driver: - self.thread_driver.stop() - + self.controller.stop() diff --git a/requirements.txt b/requirements.txt index cd93f448e28..b00a21ab495 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # -*- conf-mode -*- setuptools>=51.1.0 # Require this first, to prevent later errors # +aiosmtpd>=1.4.6 argon2-cffi>=21.3.0 # For the Argon2 password hasher option beautifulsoup4>=4.11.1 # Only used in tests bibtexparser>=1.2.0 # Only used in tests From 2652c96c8db968a021f6f95953d8ab2337aa167e Mon Sep 17 00:00:00 2001 From: rjsparks <10996692+rjsparks@users.noreply.github.com> Date: Mon, 21 Apr 2025 16:13:32 +0000 Subject: [PATCH 11/17] ci: update base image target version to 20250421T1600 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index cd0a70667c5..bdf3cff0e4a 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20250402T1611 +FROM ghcr.io/ietf-tools/datatracker-app-base:20250421T1600 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 1195fc9a0b1..bdfdb2eed1f 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20250402T1611 +20250421T1600 From 35db33961d91ada14c5744af3cf8c160e02c5614 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 22 Apr 2025 12:31:07 -0500 Subject: [PATCH 12/17] fix: lengthen liaison response_contact field (#8823) --- ...lter_liaisonstatement_response_contacts.py | 20 +++++++++++++++++++ ietf/liaisons/models.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 ietf/liaisons/migrations/0002_alter_liaisonstatement_response_contacts.py diff --git a/ietf/liaisons/migrations/0002_alter_liaisonstatement_response_contacts.py b/ietf/liaisons/migrations/0002_alter_liaisonstatement_response_contacts.py new file mode 100644 index 00000000000..ac0a11101b3 --- /dev/null +++ b/ietf/liaisons/migrations/0002_alter_liaisonstatement_response_contacts.py @@ -0,0 +1,20 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("liaisons", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="liaisonstatement", + name="response_contacts", + field=models.TextField( + blank=True, help_text="Where to send a response", max_length=1024 + ), + ), + ] diff --git a/ietf/liaisons/models.py b/ietf/liaisons/models.py index f357d6cf8ca..2ad502102c8 100644 --- a/ietf/liaisons/models.py +++ b/ietf/liaisons/models.py @@ -33,7 +33,7 @@ class LiaisonStatement(models.Model): to_groups = models.ManyToManyField(Group, blank=True, related_name='liaisonstatement_to_set') to_contacts = models.CharField(max_length=2000, help_text="Contacts at recipient group") - response_contacts = models.CharField(blank=True, max_length=255, help_text="Where to send a response") # RFC4053 + response_contacts = models.TextField(blank=True, max_length=1024, help_text="Where to send a response") # RFC4053 technical_contacts = models.CharField(blank=True, max_length=255, help_text="Who to contact for clarification") # RFC4053 action_holder_contacts = models.CharField(blank=True, max_length=255, help_text="Who makes sure action is completed") # incoming only? cc_contacts = models.TextField(blank=True) From 7be503cf1c6c02ed7715f6b72417cbc760369175 Mon Sep 17 00:00:00 2001 From: Ryan Cross Date: Wed, 23 Apr 2025 08:33:13 -0700 Subject: [PATCH 13/17] feat: use new mail archive search API for review app lookups (#8788) * feat: use new mail archive search API for review app lookups * fix: fix patch call * fix: remove url from query_data * fix: force utc timezone on datetime from archive API * fix: remove query_data['url'] from test --- ietf/doc/tests_review.py | 220 ++++++++++-------- ietf/doc/views_review.py | 22 +- ietf/review/mailarch.py | 126 ++++------ ietf/settings.py | 2 + .../templates/doc/review/complete_review.html | 4 +- 5 files changed, 176 insertions(+), 198 deletions(-) diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py index 13ddbc22ba3..9850beca753 100644 --- a/ietf/doc/tests_review.py +++ b/ietf/doc/tests_review.py @@ -3,12 +3,12 @@ from pathlib import Path -import datetime, os, shutil +import datetime import io -import tarfile, tempfile, mailbox -import email.mime.multipart, email.mime.text, email.utils +import os +import shutil -from mock import patch +from mock import patch, Mock from requests import Response from django.apps import apps @@ -650,112 +650,132 @@ def test_accept_reviewer_assignment_after_reject(self): assignment = reload_db_objects(assignment) self.assertEqual(assignment.state_id, "accepted") - def make_test_mbox_tarball(self, review_req): - mbox_path = os.path.join(self.review_dir, "testmbox.tar.gz") - with tarfile.open(mbox_path, "w:gz") as tar: - with tempfile.NamedTemporaryFile(dir=self.review_dir, suffix=".mbox") as tmp: - mbox = mailbox.mbox(tmp.name) + @patch('ietf.review.mailarch.requests.post') + def test_retrieve_messages(self, mock_post): + mock_data = { + "results": [ + { + "from": "Alice ", + "subject": "Hello", + "content": "Hi\n This is a really good document.\n", + "message_id": "abc123", + "url": "https://example.com/message", + "date": "2025-04-07T12:00:00", + } + ] + } + mock_post.return_value.json.return_value = mock_data - # plain text - msg = email.mime.text.MIMEText("Hello,\n\nI have reviewed the document and did not find any problems.\n\nJohn Doe") - msg["From"] = "John Doe " - msg["To"] = review_req.team.list_email - msg["Subject"] = "Review of {}-01".format(review_req.doc.name) - msg["Message-ID"] = email.utils.make_msgid() - msg["Archived-At"] = "" - msg["Date"] = email.utils.formatdate() - - mbox.add(msg) - - # plain text + HTML - msg = email.mime.multipart.MIMEMultipart('alternative') - msg["From"] = "John Doe II " - msg["To"] = review_req.team.list_email - msg["Subject"] = "Review of {}".format(review_req.doc.name) - msg["Message-ID"] = email.utils.make_msgid() - msg["Archived-At"] = "" - - msg.attach(email.mime.text.MIMEText("Hi!,\r\nLooks OK!\r\n-John", "plain")) - msg.attach(email.mime.text.MIMEText("

    Hi!,

    Looks OK!

    -John

    ", "html")) - mbox.add(msg) - - tmp.flush() - - tar.add(os.path.relpath(tmp.name)) + doc = WgDraftFactory(group__acronym='mars', rev='01') + review_team = ReviewTeamFactory(acronym="reviewteam", name="Review Team", type_id="review", list_email="reviewteam@ietf.org", parent=Group.objects.get(acronym="farfut")) + rev_role = RoleFactory(group=review_team, person__user__username='reviewer', person__user__email='reviewer@example.com', name_id='reviewer') + RoleFactory(group=review_team, person__user__username='reviewsecretary', person__user__email='reviewsecretary@example.com', name_id='secr') + review_req = ReviewRequestFactory(doc=doc, team=review_team, type_id='early', state_id='assigned', requested_by=rev_role.person, deadline=timezone.now() + datetime.timedelta(days=20)) + ReviewAssignmentFactory(review_request=review_req, reviewer=rev_role.person.email_set.first(), state_id='accepted') - mbox.close() + query_data = ietf.review.mailarch.construct_query_data(doc, review_team, query=None) + response = ietf.review.mailarch.retrieve_messages(query_data) - return mbox_path + self.assertEqual(len(response), 1) + self.assertEqual(response[0]['from'], 'Alice ') + self.assertEqual(response[0]['splitfrom'], ('Alice', 'alice@example.com')) + self.assertEqual(response[0]['subject'], 'Hello') + self.assertEqual(response[0]['content'], 'Hi\n This is a really good document.') + self.assertEqual(response[0]['message_id'], 'abc123') + self.assertEqual(response[0]['url'], 'https://example.com/message') + self.assertEqual(response[0]['utcdate'], ('2025-04-07', '12:00:00')) - def test_search_mail_archive(self): - doc = WgDraftFactory(group__acronym='mars',rev='01') + def test_construct_query_data(self): + doc = WgDraftFactory(group__acronym='mars', rev='01') review_team = ReviewTeamFactory(acronym="reviewteam", name="Review Team", type_id="review", list_email="reviewteam@ietf.org", parent=Group.objects.get(acronym="farfut")) - rev_role = RoleFactory(group=review_team,person__user__username='reviewer',person__user__email='reviewer@example.com',name_id='reviewer') - RoleFactory(group=review_team,person__user__username='reviewsecretary',person__user__email='reviewsecretary@example.com',name_id='secr') - review_req = ReviewRequestFactory(doc=doc,team=review_team,type_id='early',state_id='assigned',requested_by=rev_role.person,deadline=timezone.now()+datetime.timedelta(days=20)) + data = ietf.review.mailarch.construct_query_data(doc, review_team, query=None) + self.assertEqual(data['start_date'], (date_today() - datetime.timedelta(days=180)).isoformat()) + self.assertEqual(data['email_list'], 'reviewteam') + self.assertEqual(data['query_value'], doc.name) + self.assertEqual(data['query'], f'subject:({doc.name})') + self.assertEqual(data['limit'], '30') + + @patch('ietf.doc.views_review.requests.post') + def test_search_mail_archive(self, mock_post): + doc = WgDraftFactory(group__acronym='mars', rev='01') + review_team = ReviewTeamFactory(acronym="reviewteam", name="Review Team", type_id="review", list_email="reviewteam@ietf.org", parent=Group.objects.get(acronym="farfut")) + rev_role = RoleFactory(group=review_team, person__user__username='reviewer', person__user__email='reviewer@example.com', name_id='reviewer') + RoleFactory(group=review_team, person__user__username='reviewsecretary', person__user__email='reviewsecretary@example.com', name_id='secr') + review_req = ReviewRequestFactory(doc=doc, team=review_team, type_id='early', state_id='assigned', requested_by=rev_role.person, deadline=timezone.now() + datetime.timedelta(days=20)) assignment = ReviewAssignmentFactory(review_request=review_req, reviewer=rev_role.person.email_set.first(), state_id='accepted') + mock_data = { + "results": [ + { + "from": "Alice ", + "subject": "Review of {}-01".format(review_req.doc.name), + "content": "Hi\n This is a really good document.\n", + "message_id": "abc123", + "url": "https://example.com/message", + "date": "2025-04-07T12:00:00", + }, + { + "from": "Joe ", + "subject": "Review of {}".format(review_req.doc.name), + "content": "Hi\n I believe this is the best document.\n", + "message_id": "abc456", + "url": "https://example.com/message", + "date": "2025-04-07T12:00:00", + } + ] + } + response1 = Mock() + response1.json.return_value = mock_data + + response2 = Mock() + response2.json.return_value = mock_data + + response3 = Mock() + response3.json.return_value = {"results": []} + + mock_post.side_effect = [response1, response2, response3] + # test URL construction - query_urls = ietf.review.mailarch.construct_query_urls(doc, review_team) - self.assertTrue(review_req.doc.name in query_urls["query_data_url"]) + query_data = ietf.review.mailarch.construct_query_data(doc, review_team) + self.assertTrue(review_req.doc.name in query_data["query_value"]) - # test parsing - mbox_path = self.make_test_mbox_tarball(review_req) + url = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={"name": doc.name, "assignment_id": assignment.pk}) + url2 = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={"name": doc.name, "acronym": review_team.acronym}) + login_testing_unauthorized(self, "reviewsecretary", url) - try: - # mock URL generator and point it to local file - for this - # to work, the module (and not the function) must be - # imported in the view - real_fn = ietf.review.mailarch.construct_query_urls - ietf.review.mailarch.construct_query_urls = lambda doc, team, query=None: { "query_data_url": "file://" + os.path.abspath(mbox_path) } - url = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={ "name": doc.name, "assignment_id": assignment.pk }) - url2 = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={ "name": doc.name, "acronym": review_team.acronym }) - login_testing_unauthorized(self, "reviewsecretary", url) - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - messages = r.json()["messages"] - self.assertEqual(len(messages), 2) - - r = self.client.get(url2) - self.assertEqual(r.status_code, 200) - messages = r.json()["messages"] - self.assertEqual(len(messages), 2) - - today = date_today(datetime.timezone.utc) - - self.assertEqual(messages[0]["url"], "https://www.example.com/testmessage") - self.assertTrue("John Doe" in messages[0]["content"]) - self.assertEqual(messages[0]["subject"], "Review of {}-01".format(review_req.doc.name)) - self.assertEqual(messages[0]["revision_guess"], "01") - self.assertEqual(messages[0]["splitfrom"], ["John Doe", "johndoe@example.com"]) - self.assertEqual(messages[0]["utcdate"][0], today.isoformat()) - - self.assertEqual(messages[1]["url"], "https://www.example.com/testmessage2") - self.assertTrue("Looks OK" in messages[1]["content"]) - self.assertTrue("" not in messages[1]["content"]) - self.assertEqual(messages[1]["subject"], "Review of {}".format(review_req.doc.name)) - self.assertFalse('revision_guess' in messages[1]) - self.assertEqual(messages[1]["splitfrom"], ["John Doe II", "johndoe2@example.com"]) - self.assertEqual(messages[1]["utcdate"][0], "") - - - # Test failure to return mailarch results - no_result_path = os.path.join(self.review_dir, "mailarch_no_result.html") - with io.open(no_result_path, "w") as f: - f.write('Content-Type: text/html\n\n
    No results found
    ') - ietf.review.mailarch.construct_query_urls = lambda doc, team, query=None: { "query_data_url": "file://" + os.path.abspath(no_result_path) } - - url = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={ "name": doc.name, "assignment_id": assignment.pk }) - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - result = r.json() - self.assertNotIn('messages', result) - self.assertIn('No results found', result['error']) - - finally: - ietf.review.mailarch.construct_query_urls = real_fn + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + messages = r.json()["messages"] + self.assertEqual(len(messages), 2) + + r = self.client.get(url2) + self.assertEqual(r.status_code, 200) + messages = r.json()["messages"] + self.assertEqual(len(messages), 2) + + self.assertEqual(messages[0]["url"], "https://example.com/message") + self.assertTrue("Hi" in messages[0]["content"]) + self.assertEqual(messages[0]["subject"], "Review of {}-01".format(review_req.doc.name)) + self.assertEqual(messages[0]["revision_guess"], "01") + self.assertEqual(messages[0]["splitfrom"], ["Alice", "alice@example.com"]) + self.assertEqual(messages[0]["utcdate"], ['2025-04-07', '12:00:00']) + + self.assertEqual(messages[1]["url"], "https://example.com/message") + self.assertTrue("Hi" in messages[1]["content"]) + self.assertTrue("" not in messages[1]["content"]) + self.assertEqual(messages[1]["subject"], "Review of {}".format(review_req.doc.name)) + self.assertFalse('revision_guess' in messages[1]) + self.assertEqual(messages[1]["splitfrom"], ["Joe", "joe@example.com"]) + self.assertEqual(messages[1]["utcdate"], ['2025-04-07', '12:00:00']) + + # Test failure to return mailarch results + url = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={"name": doc.name, "assignment_id": assignment.pk}) + + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + result = r.json() + self.assertNotIn('messages', result) + self.assertIn('No results found', result['error']) def test_submit_unsolicited_review_choose_team(self): doc = WgDraftFactory(group__acronym='mars', rev='01') diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index 1f23c435fa8..6ac0e17791e 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -958,14 +958,14 @@ def complete_review(request, name, assignment_id=None, acronym=None): form = CompleteReviewForm(assignment, doc, team, is_reviewer, initial=initial) - mail_archive_query_urls = mailarch.construct_query_urls(doc, team) + mail_archive_query_data = mailarch.construct_query_data(doc, team) return render(request, 'doc/review/complete_review.html', { 'doc': doc, 'team': team, 'assignment': assignment, 'form': form, - 'mail_archive_query_urls': mail_archive_query_urls, + 'mail_archive_query_data': mail_archive_query_data, 'revising_review': revising_review, 'review_to': to, 'review_cc': cc, @@ -987,27 +987,25 @@ def search_mail_archive(request, name, acronym=None, assignment_id=None): if not (is_reviewer or can_manage_request): permission_denied(request, "You do not have permission to perform this action") - res = mailarch.construct_query_urls(doc, team, query=request.GET.get("query")) - if not res: - return JsonResponse({ "error": "Couldn't do lookup in mail archive - don't know where to look"}) - - MAX_RESULTS = 30 + query_data = mailarch.construct_query_data(doc, team, query=request.GET.get("query")) + if not query_data: + return JsonResponse({"error": "Couldn't do lookup in mail archive - don't know where to look"}) try: - res["messages"] = mailarch.retrieve_messages(res["query_data_url"])[:MAX_RESULTS] - for message in res["messages"]: + query_data["messages"] = mailarch.retrieve_messages(query_data) + for message in query_data["messages"]: try: revision_guess = message["subject"].split(name)[1].split('-')[1] message["revision_guess"] = revision_guess if revision_guess.isnumeric() else None except IndexError: pass except KeyError as e: - res["error"] = "No results found (%s)" % str(e) + query_data["error"] = "No results found (%s)" % str(e) except Exception as e: - res["error"] = "Retrieval from mail archive failed: %s" % str(e) + query_data["error"] = "Retrieval from mail archive failed: %s" % str(e) # raise # useful when debugging - return JsonResponse(res) + return JsonResponse(query_data) class EditReviewRequestCommentForm(forms.ModelForm): comment = forms.CharField(widget=forms.Textarea, strip=False) diff --git a/ietf/review/mailarch.py b/ietf/review/mailarch.py index c34a6079ce7..61abc83aa54 100644 --- a/ietf/review/mailarch.py +++ b/ietf/review/mailarch.py @@ -6,25 +6,18 @@ # mailarchive.ietf.org import base64 -import contextlib import datetime import email.utils import hashlib -import mailbox -import tarfile -import tempfile - -from urllib.parse import urlencode -from urllib.request import urlopen +import requests import debug # pyflakes:ignore -from pyquery import PyQuery from django.conf import settings from django.utils.encoding import force_bytes, force_str -from ietf.utils.mail import get_payload_text +from ietf.utils.log import log from ietf.utils.timezone import date_today @@ -43,7 +36,7 @@ def hash_list_message_id(list_name, msgid): sha.update(force_bytes(list_name)) return force_str(base64.urlsafe_b64encode(sha.digest()).rstrip(b"=")) -def construct_query_urls(doc, team, query=None): +def construct_query_data(doc, team, query=None): list_name = list_name_from_email(team.list_email) if not list_name: return None @@ -51,83 +44,48 @@ def construct_query_urls(doc, team, query=None): if not query: query = doc.name - encoded_query = "?" + urlencode({ - "qdr": "c", # custom time frame - "start_date": (date_today() - datetime.timedelta(days=180)).isoformat(), - "email_list": list_name, - "q": "subject:({})".format(query), - "as": "1", # this is an advanced search - }) - - return { - "query": query, - "query_url": settings.MAILING_LIST_ARCHIVE_URL + "/arch/search/" + encoded_query, - "query_data_url": settings.MAILING_LIST_ARCHIVE_URL + "/arch/export/mbox/" + encoded_query, + query_data = { + 'start_date': (date_today() - datetime.timedelta(days=180)).isoformat(), + 'email_list': list_name, + 'query_value': query, + 'query': f'subject:({query})', + 'limit': '30', } + return query_data def construct_message_url(list_name, msgid): return "{}/arch/msg/{}/{}".format(settings.MAILING_LIST_ARCHIVE_URL, list_name, hash_list_message_id(list_name, msgid)) -def retrieve_messages_from_mbox(mbox_fileobj): - """Return selected content in message from mbox from mailarch.""" - res = [] - with tempfile.NamedTemporaryFile(suffix=".mbox") as mbox_file: - # mailbox.mbox needs a path, so we need to put the contents - # into a file - mbox_data = mbox_fileobj.read() - mbox_file.write(mbox_data) - mbox_file.flush() - - mbox = mailbox.mbox(mbox_file.name, create=False) - for msg in mbox: - content = "" - - for part in msg.walk(): - if part.get_content_type() == "text/plain": - charset = part.get_content_charset() or "utf-8" - content += get_payload_text(part, default_charset=charset) - - # parse a couple of things for the front end - utcdate = None - d = email.utils.parsedate_tz(msg["Date"]) - if d: - utcdate = datetime.datetime.fromtimestamp(email.utils.mktime_tz(d), datetime.timezone.utc) - - res.append({ - "from": msg["From"], - "splitfrom": email.utils.parseaddr(msg["From"]), - "subject": msg["Subject"], - "content": content.replace("\r\n", "\n").replace("\r", "\n").strip("\n"), - "message_id": email.utils.unquote(msg["Message-ID"].strip()), - "url": email.utils.unquote(msg["Archived-At"].strip()), - "date": msg["Date"], - "utcdate": (utcdate.date().isoformat(), utcdate.time().isoformat()) if utcdate else ("", ""), - }) - mbox.close() - - return res - -def retrieve_messages(query_data_url): +def retrieve_messages(query_data): """Retrieve and return selected content from mailarch.""" - res = [] - - # This has not been rewritten to use requests.get() because get() does - # not handle file URLs out of the box, which we need for tesing - with contextlib.closing(urlopen(query_data_url, timeout=15)) as fileobj: - content_type = fileobj.info()["Content-type"] - if not content_type.startswith("application/x-tar"): - if content_type.startswith("text/html"): - r = fileobj.read(20000) - q = PyQuery(r) - div = q('div[class~="no-results"]') - if div: - raise KeyError("No results: %s -> %s" % (query_data_url, div.text(), )) - raise Exception("Export failed - this usually means no matches were found") - - with tarfile.open(fileobj=fileobj, mode='r|*') as tar: - for entry in tar: - if entry.isfile(): - mbox_fileobj = tar.extractfile(entry) - res.extend(retrieve_messages_from_mbox(mbox_fileobj)) - - return res + + headers = {'X-Api-Key': settings.MAILING_LIST_ARCHIVE_API_KEY} + try: + response = requests.post( + settings.MAILING_LIST_ARCHIVE_SEARCH_URL, + headers=headers, + json=query_data, + timeout=settings.DEFAULT_REQUESTS_TIMEOUT) + except requests.Timeout as exc: + log(f'POST request failed for [{query_data["url"]}]: {exc}') + raise RuntimeError(f'Timeout retrieving [{query_data["url"]}]') from exc + + results = [] + jresponse = response.json() + if 'results' not in jresponse or len(jresponse['results']) == 0: + raise KeyError(f'No results: {query_data["query"]}') + for msg in jresponse['results']: + # datetime is already UTC + dt = datetime.datetime.fromisoformat(msg['date']) + dt_utc = dt.replace(tzinfo=datetime.timezone.utc) + results.append({ + "from": msg["from"], + "splitfrom": email.utils.parseaddr(msg["from"]), + "subject": msg["subject"], + "content": msg["content"].replace("\r\n", "\n").replace("\r", "\n").strip("\n"), + "message_id": msg["message_id"], + "url": msg["url"], + "utcdate": (dt_utc.date().isoformat(), dt_utc.time().isoformat()), + }) + + return results diff --git a/ietf/settings.py b/ietf/settings.py index 33a2f976d95..8005aca8cc2 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -808,6 +808,8 @@ def skip_unreadable_post(record): # Mailing list info URL for lists hosted on the IETF servers MAILING_LIST_INFO_URL = "https://mailman3.%(domain)s/mailman3/lists/%(list_addr)s.%(domain)s" MAILING_LIST_ARCHIVE_URL = "https://mailarchive.ietf.org" +MAILING_LIST_ARCHIVE_SEARCH_URL = "https://mailarchive.ietf.org/api/v1/message/search/" +MAILING_LIST_ARCHIVE_API_KEY = "changeme" # Liaison Statement Tool settings (one is used in DOC_HREFS below) LIAISON_UNIVERSAL_FROM = 'Liaison Statement Management Tool ' diff --git a/ietf/templates/doc/review/complete_review.html b/ietf/templates/doc/review/complete_review.html index 091f30532e6..8aeb1486a4c 100644 --- a/ietf/templates/doc/review/complete_review.html +++ b/ietf/templates/doc/review/complete_review.html @@ -93,7 +93,7 @@

    Back {% endif %}
    - {% if mail_archive_query_urls %} + {% if mail_archive_query_data %}
    + value="{{ mail_archive_query_data.query_value }}">
    From f4209ea5a3061639f2adb9245271bb52f44a2140 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 23 Apr 2025 13:21:23 -0300 Subject: [PATCH 14/17] chore: k8s plumbing for mailarchive secret (#8827) * chore: k8s plumbing for mailarchive secret plus a typo fix * style: remove empty line * chore: hints for working with APP_API_TOKENS value --- k8s/settings_local.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index a0e197b7f5e..0e128b1b4cb 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -135,6 +135,12 @@ def _multiline_to_list(s): queue=os.environ.get("RABBITMQ_QUEUE", "dt"), ) +# mailarchive API key +_mailing_list_archive_api_key = os.environ.get("DATATRACKER_MAILING_LIST_ARCHIVE_API_KEY", None) +if _mailing_list_archive_api_key is None: + raise RuntimeError("DATATRACKER_MAILING_LIST_ARCHIVE_API_KEY must be set") +MAILING_LIST_ARCHIVE_API_KEY = _mailing_list_archive_api_key + IANA_SYNC_USERNAME = "ietfsync" IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" IANA_SYNC_PROTOCOLS_URL = "http://www.iana.org/protocols/" @@ -173,7 +179,13 @@ def _multiline_to_list(s): "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set" ) -# For APP_API_TOKENS, ccept either base64-encoded JSON or raw JSON, but not both +# For APP_API_TOKENS, accept either base64-encoded JSON or raw JSON, but not both. +# To decode / pretty-print the encoded form, run: +# base64 -d | jq . +# paste the encoded secret into stdin. Copy/paste that into an editor you trust not +# to leave a copy lying around. When done editing, copy/paste the final JSON through +# jq -c | base64 +# and copy/paste the output into the secret store. if "DATATRACKER_APP_API_TOKENS_JSON_B64" in os.environ: if "DATATRACKER_APP_API_TOKENS_JSON" in os.environ: raise RuntimeError( From 1bede712301206b808d2650e726b82d864e74d5d Mon Sep 17 00:00:00 2001 From: Ryan Cross Date: Wed, 23 Apr 2025 09:26:34 -0700 Subject: [PATCH 15/17] refactor: update secretariat dashboard style (#8140) * refactor: update secretariat dashboard style * refactor: change announcement styling * feat: hide to_custom field when not needed * fix: apply Black to python files * fix: add announcement.js to package.json * fix: move announcement playwright test to tests-legacy --------- Co-authored-by: Robert Sparks --- ietf/secr/announcement/forms.py | 137 ++++++++++++------ ietf/secr/announcement/tests.py | 109 +++++++------- ietf/secr/announcement/urls.py | 5 +- ietf/secr/announcement/views.py | 95 ++++++------ ietf/secr/templates/announcement/confirm.html | 33 ++--- ietf/secr/templates/announcement/index.html | 31 ++++ ietf/secr/templates/announcement/main.html | 36 ----- ietf/secr/templates/index.html | 33 +++++ ietf/secr/templates/main.html | 69 --------- ietf/secr/urls.py | 2 +- ietf/static/js/announcement.js | 57 ++++++++ package.json | 1 + playwright/.gitignore | 1 + .../tests-legacy/secr/announcement.spec.js | 77 ++++++++++ 14 files changed, 413 insertions(+), 273 deletions(-) create mode 100644 ietf/secr/templates/announcement/index.html delete mode 100644 ietf/secr/templates/announcement/main.html create mode 100644 ietf/secr/templates/index.html delete mode 100644 ietf/secr/templates/main.html create mode 100644 ietf/static/js/announcement.js create mode 100644 playwright/tests-legacy/secr/announcement.spec.js diff --git a/ietf/secr/announcement/forms.py b/ietf/secr/announcement/forms.py index 3fe58bdaaab..820ef79a093 100644 --- a/ietf/secr/announcement/forms.py +++ b/ietf/secr/announcement/forms.py @@ -14,30 +14,41 @@ # Globals # --------------------------------------------- -TO_LIST = ('IETF Announcement List ', - 'I-D Announcement List ', - 'RFP Announcement List ', - 'The IESG ', - 'Working Group Chairs ', - 'BOF Chairs ', - 'Other...') +TO_LIST = ( + "IETF Announcement List ", + "I-D Announcement List ", + "RFP Announcement List ", + "The IESG ", + "Working Group Chairs ", + "BOF Chairs ", + "Other...", +) # --------------------------------------------- # Helper Functions # --------------------------------------------- + def get_from_choices(user): - ''' + """ This function returns a choices tuple containing all the Announced From choices. Including leadership chairs and other entities. - ''' + """ addresses = [] - if has_role(user,'Secretariat'): - addresses = AnnouncementFrom.objects.values_list('address', flat=True).order_by('address').distinct() + if has_role(user, "Secretariat"): + addresses = ( + AnnouncementFrom.objects.values_list("address", flat=True) + .order_by("address") + .distinct() + ) else: for role in user.person.role_set.all(): - addresses.extend(AnnouncementFrom.objects.filter(name=role.name, group=role.group).values_list('address', flat=True).order_by('address')) + addresses.extend( + AnnouncementFrom.objects.filter(name=role.name, group=role.group) + .values_list("address", flat=True) + .order_by("address") + ) nomcom_choices = get_nomcom_choices(user) if nomcom_choices: @@ -45,65 +56,91 @@ def get_from_choices(user): choices = list(zip(addresses, addresses)) if len(choices) > 1: - choices.insert(0, ('', '(Choose an option)')) + choices.insert(0, ("", "(Choose an option)")) return choices def get_nomcom_choices(user): - ''' + """ Returns the list of nomcom email addresses for given user - ''' - nomcoms = Role.objects.filter(name="chair", - group__acronym__startswith="nomcom", - group__state="active", - group__type="nomcom", - person=user.person) + """ + nomcoms = Role.objects.filter( + name="chair", + group__acronym__startswith="nomcom", + group__state="active", + group__type="nomcom", + person=user.person, + ) addresses = [] for nomcom in nomcoms: year = nomcom.group.acronym[-4:] - addresses.append('NomCom Chair %s ' % (year,year)) + addresses.append("NomCom Chair %s " % (year, year)) return addresses - + def get_to_choices(): - return list(zip(TO_LIST,TO_LIST)) + return list(zip(TO_LIST, TO_LIST)) # --------------------------------------------- # Forms # --------------------------------------------- + class AnnounceForm(forms.ModelForm): - nomcom = forms.ModelChoiceField(queryset=Group.objects.filter(acronym__startswith='nomcom',type='nomcom',state='active'),required=False) + nomcom = forms.ModelChoiceField( + queryset=Group.objects.filter( + acronym__startswith="nomcom", type="nomcom", state="active" + ), + required=False, + ) to_custom = MultiEmailField(required=False) class Meta: model = Message - fields = ('nomcom', 'to','to_custom','frm','cc','bcc','reply_to','subject','body') + fields = ( + "nomcom", + "to", + "to_custom", + "frm", + "cc", + "bcc", + "reply_to", + "subject", + "body", + ) + labels = {"frm": "From"} + help_texts = { + "to": "Select name OR select Other... and enter email below", + "cc": "Use comma separated lists for emails (Cc, Bcc, Reply To)", + } def __init__(self, *args, **kwargs): - if 'hidden' in kwargs: - self.hidden = kwargs.pop('hidden') + if "hidden" in kwargs: + self.hidden = kwargs.pop("hidden") else: self.hidden = False - user = kwargs.pop('user') + user = kwargs.pop("user") person = user.person super(AnnounceForm, self).__init__(*args, **kwargs) - self.fields['to'].widget = forms.Select(choices=get_to_choices()) - self.fields['to'].help_text = 'Select name OR select Other... and enter email below' - self.fields['cc'].help_text = 'Use comma separated lists for emails (Cc, Bcc, Reply To)' - self.fields['frm'].widget = forms.Select(choices=get_from_choices(user)) - self.fields['frm'].label = 'From' - self.fields['reply_to'].required = True - self.fields['nomcom'].label = 'NomCom message:' - nomcom_roles = person.role_set.filter(group__in=self.fields['nomcom'].queryset,name='chair') - secr_roles = person.role_set.filter(group__acronym='secretariat',name='secr') + self.fields["to"].widget = forms.Select(choices=get_to_choices()) + self.fields["frm"].widget = forms.Select(choices=get_from_choices(user)) + self.fields["reply_to"].required = True + # nomcom field is defined declaratively so label and help_text must be set here + self.fields["nomcom"].label = "NomCom message:" + self.fields["nomcom"].help_text = ( + "If this is a NomCom announcement specifiy which NomCom group here" + ) + nomcom_roles = person.role_set.filter( + group__in=self.fields["nomcom"].queryset, name="chair" + ) + secr_roles = person.role_set.filter(group__acronym="secretariat", name="secr") if nomcom_roles: - self.initial['nomcom'] = nomcom_roles[0].group.pk + self.initial["nomcom"] = nomcom_roles[0].group.pk if not nomcom_roles and not secr_roles: - self.fields['nomcom'].widget = forms.HiddenInput() - + self.fields["nomcom"].widget = forms.HiddenInput() + if self.hidden: for key in list(self.fields.keys()): self.fields[key].widget = forms.HiddenInput() @@ -113,25 +150,29 @@ def clean(self): data = self.cleaned_data if self.errors: return self.cleaned_data - if data['to'] == 'Other...' and not data['to_custom']: + if data["to"] == "Other..." and not data["to_custom"]: raise forms.ValidationError('You must enter a "To" email address') - for k in ['to', 'frm', 'cc',]: + for k in [ + "to", + "frm", + "cc", + ]: data[k] = unescape(data[k]) return data def save(self, *args, **kwargs): - user = kwargs.pop('user') + user = kwargs.pop("user") message = super(AnnounceForm, self).save(commit=False) message.by = user.person - if self.cleaned_data['to'] == 'Other...': - message.to = self.cleaned_data['to_custom'] - if kwargs['commit']: + if self.cleaned_data["to"] == "Other...": + message.to = self.cleaned_data["to_custom"] + if kwargs["commit"]: message.save() # handle nomcom message - nomcom = self.cleaned_data.get('nomcom',False) + nomcom = self.cleaned_data.get("nomcom", False) if nomcom: message.related_groups.add(nomcom) - return message \ No newline at end of file + return message diff --git a/ietf/secr/announcement/tests.py b/ietf/secr/announcement/tests.py index c147c301b61..f08e824397e 100644 --- a/ietf/secr/announcement/tests.py +++ b/ietf/secr/announcement/tests.py @@ -6,7 +6,7 @@ from django.urls import reverse -import debug # pyflakes:ignore +import debug # pyflakes:ignore from ietf.utils.test_utils import TestCase from ietf.group.factories import RoleFactory @@ -17,97 +17,102 @@ from ietf.message.models import AnnouncementFrom from ietf.utils.mail import outbox, empty_outbox -SECR_USER='secretary' -WG_USER='' -AD_USER='' +SECR_USER = "secretary" +WG_USER = "" +AD_USER = "" + class SecrAnnouncementTestCase(TestCase): def setUp(self): super().setUp() - chair = RoleName.objects.get(slug='chair') - secr = RoleName.objects.get(slug='secr') - ietf = Group.objects.get(acronym='ietf') - iab = Group.objects.get(acronym='iab') - secretariat = Group.objects.get(acronym='secretariat') - AnnouncementFrom.objects.create(name=secr,group=secretariat,address='IETF Secretariat ') - AnnouncementFrom.objects.create(name=chair,group=ietf,address='IETF Chair ') - AnnouncementFrom.objects.create(name=chair,group=iab,address='IAB Chair ') + chair = RoleName.objects.get(slug="chair") + secr = RoleName.objects.get(slug="secr") + ietf = Group.objects.get(acronym="ietf") + iab = Group.objects.get(acronym="iab") + secretariat = Group.objects.get(acronym="secretariat") + AnnouncementFrom.objects.create( + name=secr, + group=secretariat, + address="IETF Secretariat ", + ) + AnnouncementFrom.objects.create( + name=chair, group=ietf, address="IETF Chair " + ) + AnnouncementFrom.objects.create( + name=chair, group=iab, address="IAB Chair " + ) def test_main(self): "Main Test" - url = reverse('ietf.secr.announcement.views.main') + url = reverse("ietf.secr.announcement.views.main") self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) - + def test_main_announce_from(self): - url = reverse('ietf.secr.announcement.views.main') + url = reverse("ietf.secr.announcement.views.main") # Secretariat self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#id_frm option')),4) + self.assertEqual(len(q("#id_frm option")), 4) # IAB Chair self.client.login(username="iab-chair", password="iab-chair+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#id_frm option')),1) - self.assertTrue('' in q('#id_frm option').val()) + self.assertEqual(len(q("#id_frm option")), 1) + self.assertTrue("" in q("#id_frm option").val()) # IETF Chair self.client.login(username="ietf-chair", password="ietf-chair+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#id_frm option')),1) - self.assertTrue('' in q('#id_frm option').val()) + self.assertEqual(len(q("#id_frm option")), 1) + self.assertTrue("" in q("#id_frm option").val()) + class UnauthorizedAnnouncementCase(TestCase): def test_unauthorized(self): "Unauthorized Test" - url = reverse('ietf.secr.announcement.views.main') - person = RoleFactory(name_id='chair',group__acronym='mars').person - self.client.login(username=person.user.username, password=person.user.username+"+password") + url = reverse("ietf.secr.announcement.views.main") + person = RoleFactory(name_id="chair", group__acronym="mars").person + self.client.login( + username=person.user.username, password=person.user.username + "+password" + ) r = self.client.get(url) self.assertEqual(r.status_code, 403) - + + class SubmitAnnouncementCase(TestCase): - def test_invalid_submit(self): - "Invalid Submit" - url = reverse('ietf.secr.announcement.views.main') - post_data = {'id_subject':''} - self.client.login(username="secretary", password="secretary+password") - r = self.client.post(url,post_data) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertTrue(len(q('form ul.errorlist')) > 0) - def test_valid_submit(self): "Valid Submit" nomcom_test_data() empty_outbox() - url = reverse('ietf.secr.announcement.views.main') - confirm_url = reverse('ietf.secr.announcement.views.confirm') - nomcom = Group.objects.get(type='nomcom') - post_data = {'nomcom': nomcom.pk, - 'to':'Other...', - 'to_custom':'rcross@amsl.com', - 'frm':'IETF Secretariat <ietf-secretariat@ietf.org>', - 'reply_to':'secretariat@ietf.org', - 'subject':'Test Subject', - 'body':'This is a test.'} + url = reverse("ietf.secr.announcement.views.main") + confirm_url = reverse("ietf.secr.announcement.views.confirm") + nomcom = Group.objects.get(type="nomcom") + post_data = { + "nomcom": nomcom.pk, + "to": "Other...", + "to_custom": "phil@example.com", + "frm": "IETF Secretariat <ietf-secretariat@ietf.org>", + "reply_to": "secretariat@ietf.org", + "subject": "Test Subject", + "body": "This is a test.", + } self.client.login(username="secretary", password="secretary+password") - response = self.client.post(url,post_data) - self.assertContains(response, 'Confirm Announcement') - response = self.client.post(confirm_url,post_data,follow=True) + response = self.client.post(url, post_data) + self.assertContains(response, "Confirm Announcement") + response = self.client.post(confirm_url, post_data, follow=True) self.assertRedirects(response, url) - self.assertEqual(len(outbox),1) - self.assertEqual(outbox[0]['subject'],'Test Subject') - self.assertEqual(outbox[0]['to'],'') - message = Message.objects.filter(by__user__username='secretary').last() - self.assertEqual(message.subject,'Test Subject') + self.assertEqual(len(outbox), 1) + self.assertEqual(outbox[0]["subject"], "Test Subject") + self.assertEqual(outbox[0]["to"], "") + message = Message.objects.filter(by__user__username="secretary").last() + self.assertEqual(message.subject, "Test Subject") self.assertTrue(nomcom in message.related_groups.all()) diff --git a/ietf/secr/announcement/urls.py b/ietf/secr/announcement/urls.py index 3c3c05a09c4..dc534f64aeb 100644 --- a/ietf/secr/announcement/urls.py +++ b/ietf/secr/announcement/urls.py @@ -1,8 +1,7 @@ - from ietf.secr.announcement import views from ietf.utils.urls import url urlpatterns = [ - url(r'^$', views.main), - url(r'^confirm/$', views.confirm), + url(r"^$", views.main), + url(r"^confirm/$", views.confirm), ] diff --git a/ietf/secr/announcement/views.py b/ietf/secr/announcement/views.py index 42de089c59c..5617ae9e6fe 100644 --- a/ietf/secr/announcement/views.py +++ b/ietf/secr/announcement/views.py @@ -18,86 +18,93 @@ # Helper Functions # ------------------------------------------------- def check_access(user): - ''' + """ This function takes a Django User object and returns true if the user has access to the Announcement app. - ''' + """ if hasattr(user, "person"): person = user.person if has_role(user, "Secretariat"): return True - + for role in person.role_set.all(): - if AnnouncementFrom.objects.filter(name=role.name,group=role.group): + if AnnouncementFrom.objects.filter(name=role.name, group=role.group): return True - if Role.objects.filter(name="chair", - group__acronym__startswith="nomcom", - group__state="active", - group__type="nomcom", - person=person): + if Role.objects.filter( + name="chair", + group__acronym__startswith="nomcom", + group__state="active", + group__type="nomcom", + person=person, + ): return True return False + # -------------------------------------------------- # STANDARD VIEW FUNCTIONS # -------------------------------------------------- # this seems to cause some kind of circular problem # @check_for_cancel(reverse('home')) @login_required -@check_for_cancel('../') +@check_for_cancel("../") def main(request): - ''' + """ Main view for Announcement tool. Authrozied users can fill out email details: header, body, etc and send. - ''' + """ if not check_access(request.user): - permission_denied(request, 'Restricted to: Secretariat, IAD, or chair of IETF, IAB, RSOC, RSE, IAOC, ISOC, NomCom.') + permission_denied( + request, + "Restricted to: Secretariat, IAD, or chair of IETF, IAB, RSOC, RSE, IAOC, ISOC, NomCom.", + ) - form = AnnounceForm(request.POST or None,user=request.user) + form = AnnounceForm(request.POST or None, user=request.user) if form.is_valid(): # recast as hidden form for next page of process form = AnnounceForm(request.POST, user=request.user, hidden=True) - if form.data['to'] == 'Other...': - to = form.data['to_custom'] + if form.data["to"] == "Other...": + to = form.data["to_custom"] else: - to = form.data['to'] + to = form.data["to"] - return render(request, 'announcement/confirm.html', { - 'message': form.data, - 'to': to, - 'form': form}, + return render( + request, + "announcement/confirm.html", + {"message": form.data, "to": to, "form": form}, ) - return render(request, 'announcement/main.html', { 'form': form} ) + return render(request, "announcement/index.html", {"form": form}) + @login_required -@check_for_cancel('../') +@check_for_cancel("../") def confirm(request): if not check_access(request.user): - permission_denied(request, 'Restricted to: Secretariat, IAD, or chair of IETF, IAB, RSOC, RSE, IAOC, ISOC, NomCom.') + permission_denied( + request, + "Restricted to: Secretariat, IAD, or chair of IETF, IAB, RSOC, RSE, IAOC, ISOC, NomCom.", + ) - if request.method == 'POST': + if request.method == "POST": form = AnnounceForm(request.POST, user=request.user) - if request.method == 'POST': - message = form.save(user=request.user,commit=True) - extra = {'Reply-To': message.get('reply_to') } - send_mail_text(None, - message.to, - message.frm, - message.subject, - message.body, - cc=message.cc, - bcc=message.bcc, - extra=extra, - ) - - messages.success(request, 'The announcement was sent.') - return redirect('ietf.secr.announcement.views.main') - - - - + if request.method == "POST": + message = form.save(user=request.user, commit=True) + extra = {"Reply-To": message.get("reply_to")} + send_mail_text( + None, + message.to, + message.frm, + message.subject, + message.body, + cc=message.cc, + bcc=message.bcc, + extra=extra, + ) + + messages.success(request, "The announcement was sent.") + return redirect("ietf.secr.announcement.views.main") diff --git a/ietf/secr/templates/announcement/confirm.html b/ietf/secr/templates/announcement/confirm.html index ddf2a6de6ed..0e1f72c54b2 100644 --- a/ietf/secr/templates/announcement/confirm.html +++ b/ietf/secr/templates/announcement/confirm.html @@ -1,22 +1,17 @@ -{% extends "base_site.html" %} +{# Copyright The IETF Trust 2024, All Rights Reserved #} +{% extends "base.html" %} {% load static %} - +{% load ietf_filters %} +{% load django_bootstrap5 %} {% block title %}Announcement{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Announcement -{% endblock %} - {% block content %} +

    Announcement

    +

    Confirm Announcement

    -
    {% csrf_token %} + {% csrf_token %} 
     To: {{ to }}
@@ -29,15 +24,13 @@ 
    Confirm Announcement
    
 {{ message.body }}
    - {{ form }} -
    -
      -
      • -
      • -
      • -
    -
    + {% bootstrap_form form %} +
    + + + +
    diff --git a/ietf/secr/templates/announcement/index.html b/ietf/secr/templates/announcement/index.html new file mode 100644 index 00000000000..ad7226e3bc1 --- /dev/null +++ b/ietf/secr/templates/announcement/index.html @@ -0,0 +1,31 @@ +{# Copyright The IETF Trust 2024, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load ietf_filters %} +{% load django_bootstrap5 %} +{% block title %}Announcement{% endblock %} +{% block content %} +

    Announcement

    + {% if form.non_field_errors %}
    {{ form.non_field_errors }}
    {% endif %} + +
    + {% csrf_token %} + {% bootstrap_field form.nomcom layout='horizontal' %} + {% bootstrap_field form.to layout='horizontal' %} + {% bootstrap_field form.to_custom layout='horizontal' %} + {% bootstrap_field form.frm layout='horizontal' %} + {% bootstrap_field form.cc layout='horizontal' %} + {% bootstrap_field form.bcc layout='horizontal' %} + {% bootstrap_field form.reply_to layout='horizontal' %} + {% bootstrap_field form.subject layout='horizontal' %} + {% bootstrap_field form.body layout='horizontal' %} + + + Cancel +
    + +{% endblock %} +{% block js %} + +{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/announcement/main.html b/ietf/secr/templates/announcement/main.html deleted file mode 100644 index c88b4a2406c..00000000000 --- a/ietf/secr/templates/announcement/main.html +++ /dev/null @@ -1,36 +0,0 @@ -{% extends "base_site.html" %} - -{% block title %}Announcement{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Announcement -{% endblock %} - -{% block content %} - -
    -

    Announcement

    - -
    {% csrf_token %} - - - {% if form.non_field_errors %}{{ form.non_field_errors }}{% endif %} - {% for field in form.visible_fields %} - - - - - {% endfor %} - -
    {{ field.label_tag }}{% if field.field.required %} *{% endif %}{{ field.errors }}{{ field }}{% if field.help_text %}
    {{ field.help_text }}{% endif %}
    -
    -
      -
      • -
      • -
    -
    - -
    -
    - -{% endblock %} diff --git a/ietf/secr/templates/index.html b/ietf/secr/templates/index.html new file mode 100644 index 00000000000..05fa3db41f0 --- /dev/null +++ b/ietf/secr/templates/index.html @@ -0,0 +1,33 @@ +{# Copyright The IETF Trust 2007, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load ietf_filters %} +{% block title %}Secretariat Dashboard{% endblock %} +{% block content %} +

    Secretariat Dashboard

    +
    + {% if user|has_role:"Secretariat" %} +

    IESG

    + + +

    IDs and WGs Process

    + + +

    Meetings and Proceedings

    + + {% else %} + + {% endif %} +
    +{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/main.html b/ietf/secr/templates/main.html deleted file mode 100644 index 42d6e8f6a13..00000000000 --- a/ietf/secr/templates/main.html +++ /dev/null @@ -1,69 +0,0 @@ -{% extends "base_site.html" %} -{% load ietf_filters %} - -{% block content %} -
    - - {% if user|has_role:"Secretariat" %} - - - - - - - - - - - - -
    -

    IESG

    - -     		-

    IDs and WGs Process

    - -
    -

    Meetings and Proceedings

    - -
    - - {% else %} - - - - - - - - - - - - -
    -

    Section 1

    - -     		-

    Section 2

    -
    -

    Section 3

    -     		-

    Section 4

    -
    - - {% endif %} - -
    -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/urls.py b/ietf/secr/urls.py index 0ce14a449aa..4a3e5b0363f 100644 --- a/ietf/secr/urls.py +++ b/ietf/secr/urls.py @@ -2,7 +2,7 @@ from django.views.generic import TemplateView urlpatterns = [ - re_path(r'^$', TemplateView.as_view(template_name='main.html')), + re_path(r'^$', TemplateView.as_view(template_name='index.html'), name='ietf.secr'), re_path(r'^announcement/', include('ietf.secr.announcement.urls')), re_path(r'^meetings/', include('ietf.secr.meetings.urls')), re_path(r'^rolodex/', include('ietf.secr.rolodex.urls')), diff --git a/ietf/static/js/announcement.js b/ietf/static/js/announcement.js new file mode 100644 index 00000000000..95465120fa8 --- /dev/null +++ b/ietf/static/js/announcement.js @@ -0,0 +1,57 @@ +const announcementApp = (function() { + 'use strict'; + return { + // functions for Announcement + checkToField: function() { + document.documentElement.scrollTop = 0; // For most browsers + const toField = document.getElementById('id_to'); + const toCustomInput = document.getElementById('id_to_custom'); + const toCustomDiv = toCustomInput.closest('div.row'); + + if (toField.value === 'Other...') { + toCustomDiv.style.display = 'flex'; // Show the custom field + } else { + toCustomDiv.style.display = 'none'; // Hide the custom field + toCustomInput.value = ''; // Optionally clear the input value if hidden + } + } + }; +})(); + +// Extra care is required to ensure the back button +// works properly for the optional to_custom field. +// Take the case when a user selects "Other..." for +// "To" field. The "To custom" field appears and they +// enter a new address there. +// In Chrome, when the form is submitted and then the user +// uses the back button (or browser back), the page loads +// from bfcache then the javascript DOMContentLoaded event +// handler is run, hiding the empty to_custom field, THEN the +// browser autofills the form fields. Because to_submit +// is now hidden it does not get a value. This is a very +// bad experience for the user because the to_custom field +// was unexpectedly cleared and hidden. If they notice this +// they would need to know to first select another "To" +// option, then select "Other..." again just to get the +// to_custom field visible so they can re-enter the custom +// address. +// The solution is to use setTimeout to run checkToField +// after a short delay, giving the browser time to autofill +// the form fields before it checks to see if the to_custom +// field is empty and hides it. + +document.addEventListener('DOMContentLoaded', function() { + // Run the visibility check after allowing cache to populate values + setTimeout(announcementApp.checkToField, 300); + + const toField = document.getElementById('id_to'); + toField.addEventListener('change', announcementApp.checkToField); +}); + +// Handle back/forward navigation with pageshow +window.addEventListener('pageshow', function(event) { + if (event.persisted) { + // Then apply visibility logic after cache restoration + setTimeout(announcementApp.checkToField, 300); + } +}); \ No newline at end of file diff --git a/package.json b/package.json index 6d1591ca151..e3e89288e77 100644 --- a/package.json +++ b/package.json @@ -114,6 +114,7 @@ "ietf/static/js/add_session_recordings.js", "ietf/static/js/agenda_filter.js", "ietf/static/js/agenda_materials.js", + "ietf/static/js/announcement.js", "ietf/static/js/complete-review.js", "ietf/static/js/create_timeslot.js", "ietf/static/js/create_timeslot.js", diff --git a/playwright/.gitignore b/playwright/.gitignore index 75e854d8dcf..f38d036a792 100644 --- a/playwright/.gitignore +++ b/playwright/.gitignore @@ -2,3 +2,4 @@ node_modules/ /test-results/ /playwright-report/ /playwright/.cache/ +auth.json \ No newline at end of file diff --git a/playwright/tests-legacy/secr/announcement.spec.js b/playwright/tests-legacy/secr/announcement.spec.js new file mode 100644 index 00000000000..4dbbc25a813 --- /dev/null +++ b/playwright/tests-legacy/secr/announcement.spec.js @@ -0,0 +1,77 @@ +const { test, expect } = require('@playwright/test') +const viewports = require('../../helpers/viewports') +const { setTimeout } = require('timers/promises') + +// ==================================================================== +// ANNOUNCEMENT | DESKTOP viewport +// ==================================================================== + +test.describe('desktop', () => { + + test.beforeAll(async ({ browser }) => { + const context = await browser.newContext(); + const page = await context.newPage(); + + await page.goto('/accounts/login/'); + + await page.fill('input#id_username', 'glen'); + await page.fill('input#id_password', 'password'); + + await page.click('button[type="submit"]'); + await page.waitForURL('/accounts/profile/'); + + await context.storageState({ path: 'auth.json' }); + + await context.close(); + }); + + test.beforeEach(async ({ browser }) => { + // Reuse the authentication state in each test + const context = await browser.newContext({ storageState: 'auth.json' }); + const page = await context.newPage(); + await page.setViewportSize({ + width: viewports.desktop[0], + height: viewports.desktop[1] + }) + await page.goto(`/secr/announcement/`); + await page.locator('h1:text("Announcement")').waitFor({ state: 'visible' }) + await setTimeout(500) + // Attach the page to the test context + test.info().page = page; + }) + + test('show to custom', async () => { + const page = test.info().page; + + // to_custom should initially be hidden + const element = page.locator('#id_to_custom'); + await expect(element).toBeHidden(); + await page.selectOption('select#id_to', 'Other...'); + await expect(element).toBeVisible(); + }) + + test('back button', async () => { + const page = test.info().page; + + const element = page.locator('#id_to_custom'); + await page.selectOption('select#id_to', 'Other...'); + await expect(element).toBeVisible(); + await page.fill('input#id_to_custom', 'custom@example.com'); + await page.selectOption('select#id_frm', 'IETF Chair '); + await page.fill('input#id_reply_to', 'greg@example.com'); + await page.fill('input#id_subject', 'About Stuff'); + await page.fill('textarea#id_body', 'This is the stuff'); + + await page.click('text="Continue"'); + const h2Locator = page.locator('h2:text("Confirm Announcement")'); + await h2Locator.waitFor({ state: 'visible' }); + + // click back button and check to_custom + await page.click('text="Back"'); + const subjectLocator = page.locator('input#id_subject'); + await subjectLocator.waitFor({ state: 'visible' }); + await expect(element).toBeVisible(); + await expect(element).toHaveValue('custom@example.com'); + }) + +}) \ No newline at end of file From 1a309b7464161e5e7e590eecacef96d752af02a0 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Wed, 23 Apr 2025 14:45:45 -0400 Subject: [PATCH 16/17] ci: Revert build.yml s3 sync command change --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e5c627c950b..36439aa6cfd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -235,7 +235,7 @@ jobs: docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh echo "Pushing statics..." cd static - aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors --checksum-algorithm CRC32 + aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors - name: Augment dockerignore for docker image build env: From 3c8f9727acc4825ec33816d77275b2793e9a9713 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 24 Apr 2025 11:43:23 -0300 Subject: [PATCH 17/17] chore: get django out of the forwarded header business (#8634) --- k8s/settings_local.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 0e128b1b4cb..0f4ecaa631c 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -19,13 +19,6 @@ def _multiline_to_list(s): # Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env! SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development") -# Use X-Forwarded-Proto to determine request.is_secure(). This relies on CloudFlare overwriting the -# value of the header if an incoming request sets it, which it does: -# https://developers.cloudflare.com/fundamentals/reference/http-request-headers/#x-forwarded-proto -# See also, especially the warnings: -# https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header -SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") - # Secrets _SECRET_KEY = os.environ.get("DATATRACKER_DJANGO_SECRET_KEY", None) if _SECRET_KEY is not None: