From 23e5307d2e097ae32749f74788ad1eff59cd6042 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 4 Sep 2024 01:20:36 +1200 Subject: [PATCH 001/601] feat: Use meetecho-player.ietf.org for session recording (#7873) --- ietf/meeting/models.py | 15 ++++--- ietf/meeting/tests_models.py | 44 +++++++++++++++++++ ietf/settings.py | 2 +- .../meeting/interim_session_buttons.html | 9 ++-- 4 files changed, 60 insertions(+), 10 deletions(-) diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 693cb99dfd..3470679327 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -1321,11 +1321,16 @@ def onsite_tool_url(self): return None def session_recording_url(self): - url = getattr(settings, "MEETECHO_SESSION_RECORDING_URL", "") - if self.meeting.type.slug == "ietf" and self.has_onsite_tool and url: - self.group.acronym_upper = self.group.acronym.upper() - return url.format(session=self) - return None + url_formatter = getattr(settings, "MEETECHO_SESSION_RECORDING_URL", "") + url = None + if url_formatter and self.video_stream_url: + if self.meeting.type.slug == "ietf" and self.has_onsite_tool: + session_label = f"IETF{self.meeting.number}-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}" + else: + session_label = f"IETF-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}" + url = url_formatter.format(session_label=session_label) + + return url class SchedulingEvent(models.Model): diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py index 0ccd462715..ff874100dc 100644 --- a/ietf/meeting/tests_models.py +++ b/ietf/meeting/tests_models.py @@ -146,3 +146,47 @@ def test_chat_room_name(self): self.assertEqual(session.chat_room_name(), 'plenary') session.chat_room = 'fnord' self.assertEqual(session.chat_room_name(), 'fnord') + + def test_session_recording_url(self): + group_acronym = "foobar" + meeting_date = datetime.date.today() + meeting_number = 123 + + # IETF meeting + session = SessionFactory( + meeting__type_id='ietf', + meeting__date=meeting_date, + group__acronym=group_acronym, + meeting__number=meeting_number, + ) + with override_settings(): + if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"): + del settings.MEETECHO_SESSION_RECORDING_URL + self.assertIsNone(session.session_recording_url()) + + settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com" + self.assertEqual(session.session_recording_url(), "http://player.example.com") + + settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}" + self.assertIn(f"IETF{meeting_number}-{group_acronym.upper()}", session.session_recording_url()) + self.assertIn(f"{meeting_date.strftime('%Y%m%d')}", session.session_recording_url()) + self.assertTrue(session.session_recording_url().startswith("http://player.example.com")) + + # interim meeting + session = SessionFactory( + meeting__type_id='interim', + meeting__date=meeting_date, + group__acronym=group_acronym, + ) + with override_settings(): + if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"): + del settings.MEETECHO_SESSION_RECORDING_URL + self.assertIsNone(session.session_recording_url()) + + settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com" + self.assertEqual(session.session_recording_url(), "http://player.example.com") + + settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}" + self.assertIn(f"IETF-{group_acronym.upper()}", session.session_recording_url()) + self.assertIn(f"{meeting_date.strftime('%Y%m%d')}", session.session_recording_url()) + self.assertTrue(session.session_recording_url().startswith("http://player.example.com")) diff --git a/ietf/settings.py b/ietf/settings.py index db53efe0a5..1bb7a122c3 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1174,7 +1174,7 @@ def skip_unreadable_post(record): MEETECHO_ONSITE_TOOL_URL = "https://meetings.conf.meetecho.com/onsite{session.meeting.number}/?session={session.pk}" MEETECHO_VIDEO_STREAM_URL = "https://meetings.conf.meetecho.com/ietf{session.meeting.number}/?session={session.pk}" MEETECHO_AUDIO_STREAM_URL = "https://mp3.conf.meetecho.com/ietf{session.meeting.number}/{session.pk}.m3u" -MEETECHO_SESSION_RECORDING_URL = "https://www.meetecho.com/ietf{session.meeting.number}/recordings#{session.group.acronym_upper}" +MEETECHO_SESSION_RECORDING_URL = "https://meetecho-player.ietf.org/playout/?session={session_label}" # Put the production SECRET_KEY in settings_local.py, and also any other # sensitive or site-specific changes. DO NOT commit settings_local.py to svn. diff --git a/ietf/templates/meeting/interim_session_buttons.html b/ietf/templates/meeting/interim_session_buttons.html index a32f4345c9..23263b9859 100644 --- a/ietf/templates/meeting/interim_session_buttons.html +++ b/ietf/templates/meeting/interim_session_buttons.html @@ -146,17 +146,18 @@ {% endif %} {% endwith %} {% endfor %} - {% elif session.video_stream_url %} + {% elif show_empty %} + {# #} + {% endif %} + {% if session.session_recording_url %} - {% elif show_empty %} - {# #} {% endif %} {% endwith %} {% endif %} -{% endwith %} \ No newline at end of file +{% endwith %} From 061c89f3b559829d0d1e20ef6d81c33542906ff7 Mon Sep 17 00:00:00 2001 From: Seonghyeon Cho Date: Wed, 4 Sep 2024 00:46:34 +0900 Subject: [PATCH 002/601] fix: Missing button text for PostScript in RFC (#7889) Resolves #7879 Signed-off-by: Seonghyeon Cho --- ietf/doc/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index a98b46cb50..74000e598b 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -1046,6 +1046,8 @@ def build_file_urls(doc: Union[Document, DocHistory]): file_urls = [] for t in found_types: + if t == "ps": # Postscript might have been submitted but should not be displayed in the list of URLs + continue label = "plain text" if t == "txt" else t file_urls.append((label, base + doc.name + "." + t)) From b6f8ede98a7c3f18210cc72907b7a5de2a68ef51 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 3 Sep 2024 19:24:26 -0300 Subject: [PATCH 003/601] feat: is_authenticated request logging + cleanup (#7893) * chore: nginx log is s, not ms * chore: log seconds from gunicorn too * chore: drop X-Real-IP header / log * style: Black * style: single -> double quotes * feat: add is-authenticated header * feat: log is-authenticated header * chore: update nginx-auth.conf to match --- ietf/middleware.py | 46 ++++++++++++++++++++++++++++++-------- ietf/settings.py | 35 +++++++++++++++-------------- ietf/utils/jsonlogger.py | 4 ++-- k8s/nginx-auth.conf | 2 +- k8s/nginx-datatracker.conf | 2 +- k8s/nginx-logging.conf | 2 +- 6 files changed, 60 insertions(+), 31 deletions(-) diff --git a/ietf/middleware.py b/ietf/middleware.py index 48146abf5e..a4b7a0d24c 100644 --- a/ietf/middleware.py +++ b/ietf/middleware.py @@ -17,45 +17,61 @@ def sql_log_middleware(get_response): def sql_log(request): response = get_response(request) for q in connection.queries: - if re.match('(update|insert)', q['sql'], re.IGNORECASE): - log(q['sql']) + if re.match("(update|insert)", q["sql"], re.IGNORECASE): + log(q["sql"]) return response + return sql_log + class SMTPExceptionMiddleware(object): def __init__(self, get_response): self.get_response = get_response + def __call__(self, request): return self.get_response(request) + def process_exception(self, request, exception): if isinstance(exception, smtplib.SMTPException): (extype, value, tb) = log_smtp_exception(exception) - return render(request, 'email_failed.html', - {'exception': extype, 'args': value, 'traceback': "".join(tb)} ) + return render( + request, + "email_failed.html", + {"exception": extype, "args": value, "traceback": "".join(tb)}, + ) return None + class Utf8ExceptionMiddleware(object): def __init__(self, get_response): self.get_response = get_response + def __call__(self, request): return self.get_response(request) + def process_exception(self, request, exception): if isinstance(exception, OperationalError): extype, e, tb = exc_parts() if e.args[0] == 1366: log("Database 4-byte utf8 exception: %s: %s" % (extype, e)) - return render(request, 'utf8_4byte_failed.html', - {'exception': extype, 'args': e.args, 'traceback': "".join(tb)} ) + return render( + request, + "utf8_4byte_failed.html", + {"exception": extype, "args": e.args, "traceback": "".join(tb)}, + ) return None + def redirect_trailing_period_middleware(get_response): def redirect_trailing_period(request): response = get_response(request) if response.status_code == 404 and request.path.endswith("."): return HttpResponsePermanentRedirect(request.path.rstrip(".")) return response + return redirect_trailing_period + def unicode_nfkc_normalization_middleware(get_response): def unicode_nfkc_normalization(request): """Do Unicode NFKC normalization to turn ligatures into individual characters. @@ -65,9 +81,21 @@ def unicode_nfkc_normalization(request): There are probably other elements of a request which may need this normalization too, but let's put that in as it comes up, rather than guess ahead. """ - request.META["PATH_INFO"] = unicodedata.normalize('NFKC', request.META["PATH_INFO"]) - request.path_info = unicodedata.normalize('NFKC', request.path_info) + request.META["PATH_INFO"] = unicodedata.normalize( + "NFKC", request.META["PATH_INFO"] + ) + request.path_info = unicodedata.normalize("NFKC", request.path_info) response = get_response(request) return response + return unicode_nfkc_normalization - + + +def is_authenticated_header_middleware(get_response): + """Middleware to add an is-authenticated header to the response""" + def add_header(request): + response = get_response(request) + response["X-Datatracker-Is-Authenticated"] = "yes" if request.user.is_authenticated else "no" + return response + + return add_header diff --git a/ietf/settings.py b/ietf/settings.py index 1bb7a122c3..a1a7fee102 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -401,24 +401,25 @@ def skip_unreadable_post(record): MIDDLEWARE = [ - 'django.middleware.csrf.CsrfViewMiddleware', - 'corsheaders.middleware.CorsMiddleware', # see docs on CORS_REPLACE_HTTPS_REFERER before using it - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.http.ConditionalGetMiddleware', - 'simple_history.middleware.HistoryRequestMiddleware', + "django.middleware.csrf.CsrfViewMiddleware", + "corsheaders.middleware.CorsMiddleware", # see docs on CORS_REPLACE_HTTPS_REFERER before using it + "django.middleware.common.CommonMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.http.ConditionalGetMiddleware", + "simple_history.middleware.HistoryRequestMiddleware", # comment in this to get logging of SQL insert and update statements: - #'ietf.middleware.sql_log_middleware', - 'ietf.middleware.SMTPExceptionMiddleware', - 'ietf.middleware.Utf8ExceptionMiddleware', - 'ietf.middleware.redirect_trailing_period_middleware', - 'django_referrer_policy.middleware.ReferrerPolicyMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', - # 'csp.middleware.CSPMiddleware', - 'ietf.middleware.unicode_nfkc_normalization_middleware', + #"ietf.middleware.sql_log_middleware", + "ietf.middleware.SMTPExceptionMiddleware", + "ietf.middleware.Utf8ExceptionMiddleware", + "ietf.middleware.redirect_trailing_period_middleware", + "django_referrer_policy.middleware.ReferrerPolicyMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django.middleware.security.SecurityMiddleware", + #"csp.middleware.CSPMiddleware", + "ietf.middleware.unicode_nfkc_normalization_middleware", + "ietf.middleware.is_authenticated_header_middleware", ] ROOT_URLCONF = 'ietf.urls' diff --git a/ietf/utils/jsonlogger.py b/ietf/utils/jsonlogger.py index 9c7949fd58..c383ba310f 100644 --- a/ietf/utils/jsonlogger.py +++ b/ietf/utils/jsonlogger.py @@ -23,12 +23,12 @@ def add_fields(self, log_record, record, message_dict): log_record.setdefault("referer", record.args["f"]) log_record.setdefault("user_agent", record.args["a"]) log_record.setdefault("len_bytes", record.args["B"]) - log_record.setdefault("duration_ms", record.args["M"]) + log_record.setdefault("duration_s", record.args["L"]) # decimal seconds log_record.setdefault("host", record.args["{host}i"]) log_record.setdefault("x_request_start", record.args["{x-request-start}i"]) - log_record.setdefault("x_real_ip", record.args["{x-real-ip}i"]) log_record.setdefault("x_forwarded_for", record.args["{x-forwarded-for}i"]) log_record.setdefault("x_forwarded_proto", record.args["{x-forwarded-proto}i"]) log_record.setdefault("cf_connecting_ip", record.args["{cf-connecting-ip}i"]) log_record.setdefault("cf_connecting_ipv6", record.args["{cf-connecting-ipv6}i"]) log_record.setdefault("cf_ray", record.args["{cf-ray}i"]) + log_record.setdefault("is_authenticated", record.args["{x-datatracker-is-authenticated}i"]) diff --git a/k8s/nginx-auth.conf b/k8s/nginx-auth.conf index a38b8f50c7..95aa838064 100644 --- a/k8s/nginx-auth.conf +++ b/k8s/nginx-auth.conf @@ -32,7 +32,7 @@ server { proxy_set_header Connection close; proxy_set_header X-Request-Start "t=$${keepempty}msec"; proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $${keepempty}remote_addr; + proxy_hide_header X-Datatracker-Is-Authenticated; # hide this from the outside world proxy_pass http://localhost:8000; # Set timeouts longer than Cloudflare proxy limits proxy_connect_timeout 60; # nginx default (Cf = 15) diff --git a/k8s/nginx-datatracker.conf b/k8s/nginx-datatracker.conf index 7c0dc85fd0..882d7563c2 100644 --- a/k8s/nginx-datatracker.conf +++ b/k8s/nginx-datatracker.conf @@ -21,7 +21,7 @@ server { proxy_set_header Connection close; proxy_set_header X-Request-Start "t=$${keepempty}msec"; proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $${keepempty}remote_addr; + proxy_hide_header X-Datatracker-Is-Authenticated; # hide this from the outside world proxy_pass http://localhost:8000; # Set timeouts longer than Cloudflare proxy limits proxy_connect_timeout 60; # nginx default (Cf = 15) diff --git a/k8s/nginx-logging.conf b/k8s/nginx-logging.conf index 0938b0530e..3c4ade4614 100644 --- a/k8s/nginx-logging.conf +++ b/k8s/nginx-logging.conf @@ -9,7 +9,7 @@ log_format ietfjson escape=json '"method":"$${keepempty}request_method",' '"status":"$${keepempty}status",' '"len_bytes":"$${keepempty}body_bytes_sent",' - '"duration_ms":"$${keepempty}request_time",' + '"duration_s":"$${keepempty}request_time",' '"referer":"$${keepempty}http_referer",' '"user_agent":"$${keepempty}http_user_agent",' '"x_forwarded_for":"$${keepempty}http_x_forwarded_for",' From 36847428d5991c75cbc96799f68ff327af7b89f8 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 4 Sep 2024 17:39:03 -0300 Subject: [PATCH 004/601] fix: label > 26 sessions per group (#7599) * fix: label > 26 sessions correctly * test: test new helper --------- Co-authored-by: Robert Sparks --- ietf/meeting/models.py | 21 ++++++++++++++++----- ietf/meeting/tests_models.py | 9 +++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 3470679327..01b695bcea 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -1205,19 +1205,30 @@ def special_request_token(self): else: return "" + @staticmethod + def _alpha_str(n: int): + """Convert integer to string of a-z characters (a, b, c, ..., aa, ab, ...)""" + chars = [] + while True: + chars.append(string.ascii_lowercase[n % 26]) + n //= 26 + # for 2nd letter and beyond, 0 means end the string + if n == 0: + break + # beyond the first letter, no need to represent a 0, so decrement + n -= 1 + return "".join(chars[::-1]) + def docname_token(self): sess_mtg = Session.objects.filter(meeting=self.meeting, group=self.group).order_by('pk') index = list(sess_mtg).index(self) - return 'sess%s' % (string.ascii_lowercase[index]) + return f"sess{self._alpha_str(index)}" def docname_token_only_for_multiple(self): sess_mtg = Session.objects.filter(meeting=self.meeting, group=self.group).order_by('pk') if len(list(sess_mtg)) > 1: index = list(sess_mtg).index(self) - if index < 26: - token = 'sess%s' % (string.ascii_lowercase[index]) - else: - token = 'sess%s%s' % (string.ascii_lowercase[index//26],string.ascii_lowercase[index%26]) + token = f"sess{self._alpha_str(index)}" return token return None diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py index ff874100dc..0a089ee9e8 100644 --- a/ietf/meeting/tests_models.py +++ b/ietf/meeting/tests_models.py @@ -10,6 +10,7 @@ from ietf.group.factories import GroupFactory, GroupHistoryFactory from ietf.meeting.factories import MeetingFactory, SessionFactory, AttendedFactory, SessionPresentationFactory +from ietf.meeting.models import Session from ietf.stats.factories import MeetingRegistrationFactory from ietf.utils.test_utils import TestCase from ietf.utils.timezone import date_today, datetime_today @@ -147,6 +148,14 @@ def test_chat_room_name(self): session.chat_room = 'fnord' self.assertEqual(session.chat_room_name(), 'fnord') + def test_alpha_str(self): + self.assertEqual(Session._alpha_str(0), "a") + self.assertEqual(Session._alpha_str(1), "b") + self.assertEqual(Session._alpha_str(25), "z") + self.assertEqual(Session._alpha_str(26), "aa") + self.assertEqual(Session._alpha_str(27 * 26 - 1), "zz") + self.assertEqual(Session._alpha_str(27 * 26), "aaa") + def test_session_recording_url(self): group_acronym = "foobar" meeting_date = datetime.date.today() From 2a6fd3e1969f52c65089c024e1188c8a07bcb4a3 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 5 Sep 2024 12:39:35 -0300 Subject: [PATCH 005/601] ci: remove auth on port 8080 (#7903) It's now on port 80 --- k8s/auth.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/k8s/auth.yaml b/k8s/auth.yaml index c35cdc8ac2..c92ed05163 100644 --- a/k8s/auth.yaml +++ b/k8s/auth.yaml @@ -144,9 +144,5 @@ spec: targetPort: http protocol: TCP name: http - - port: 8080 - targetPort: http - protocol: TCP - name: http-old selector: app: auth From cb25831a2a33a97d0b5cf0ee2a0ba29af51c887b Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 5 Sep 2024 10:43:43 -0500 Subject: [PATCH 006/601] feat: total ids, pre-pubreq counts and pages left to ballot on on the AD dashboard (#7813) * feat: Total ids on IESG dashboard * IESG I-D code comments * Using Robert's query forIESG dashboard total_ids * Hiding columns in later IESG Dashboard tables * Changing IESG dashboard var name to match column table * Updating IESG pre_pubreqquery * IESG dashboard prepub req safeParser and graphs * IESG dashboard fixing Playwright API usage * IESG dashboard fixing Playwright API usage (2) * Updating .gitignore for /geckodriver.log * IESG ad test title * feat: pages left to ballot on [WIP] * Adding geckodriver.log to gitignore * [WIP] pages left to ballot on * integrating pages left to ballot on WIP * Tests for ad pages remaining * Setting states to test ballot items * refactor ad_pages_left_to_ballot_on count logic * WIP tests for pages left to ballot on * chore: remove whitespace change * fix: look into the BallotPositionDocEventObject * chore: remove prints * fix: restructure test * style: fix js code styling * fix: only show graph for ADs/Secretariat --------- Co-authored-by: Matthew Holloway Co-authored-by: holloway Co-authored-by: Nicolas Giard Co-authored-by: Matthew Holloway --- .gitignore | 1 + ietf/doc/views_search.py | 23 +++ ietf/iesg/tests.py | 68 +++++- ietf/iesg/utils.py | 22 +- ietf/iesg/views.py | 8 +- ietf/templates/doc/ad_list.html | 241 +++++++++++++++------- ietf/templates/iesg/agenda_documents.html | 7 +- playwright/tests-legacy/docs/ad.spec.js | 26 +++ 8 files changed, 307 insertions(+), 89 deletions(-) create mode 100644 playwright/tests-legacy/docs/ad.spec.js diff --git a/.gitignore b/.gitignore index c25e6b5bfe..84bc800e3b 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ datatracker.sublime-workspace /docker/docker-compose.extend-custom.yml /env /ghostdriver.log +/geckodriver.log /htmlcov /ietf/static/dist-neue /latest-coverage.json diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 2ef4ee83e6..528fb05a22 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -485,6 +485,29 @@ def _state_to_doc_type(state): ) ad.buckets = copy.deepcopy(bucket_template) + # https://github.com/ietf-tools/datatracker/issues/4577 + docs_via_group_ad = Document.objects.exclude( + group__acronym="none" + ).filter( + group__role__name="ad", + group__role__person=ad + ).filter( + states__type="draft-stream-ietf", + states__slug__in=["wg-doc","wg-lc","waiting-for-implementation","chair-w","writeupw"] + ) + + doc_for_ad = Document.objects.filter(ad=ad) + + ad.pre_pubreq = (docs_via_group_ad | doc_for_ad).filter( + type="draft" + ).filter( + states__type="draft", + states__slug="active" + ).filter( + states__type="draft-iesg", + states__slug="idexists" + ).distinct().count() + for doc in Document.objects.exclude(type_id="rfc").filter(ad=ad): dt = doc_type(doc) state = doc_state(doc) diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py index 4579316f22..8438cb44dd 100644 --- a/ietf/iesg/tests.py +++ b/ietf/iesg/tests.py @@ -18,7 +18,7 @@ from ietf.doc.models import DocEvent, BallotPositionDocEvent, TelechatDocEvent from ietf.doc.models import Document, State, RelatedDocument -from ietf.doc.factories import WgDraftFactory, IndividualDraftFactory, ConflictReviewFactory, BaseDocumentFactory, CharterFactory, WgRfcFactory, IndividualRfcFactory +from ietf.doc.factories import BallotDocEventFactory, BallotPositionDocEventFactory, TelechatDocEventFactory, WgDraftFactory, IndividualDraftFactory, ConflictReviewFactory, BaseDocumentFactory, CharterFactory, WgRfcFactory, IndividualRfcFactory from ietf.doc.utils import create_ballot_if_not_open from ietf.group.factories import RoleFactory, GroupFactory, DatedGroupMilestoneFactory, DatelessGroupMilestoneFactory from ietf.group.models import Group, GroupMilestone, Role @@ -30,7 +30,6 @@ from ietf.iesg.factories import IESGMgmtItemFactory, TelechatAgendaContentFactory from ietf.utils.timezone import date_today, DEADLINE_TZINFO - class IESGTests(TestCase): def test_feed(self): draft = WgDraftFactory(states=[('draft','active'),('draft-iesg','iesg-eva')],ad=Person.objects.get(user__username='ad')) @@ -509,12 +508,13 @@ def test_agenda_documents_txt(self): def test_agenda_documents(self): url = urlreverse("ietf.iesg.views.agenda_documents") r = self.client.get(url) + self.assertEqual(r.status_code, 200) for k, d in self.telechat_docs.items(): self.assertContains(r, d.name, msg_prefix="%s '%s' not in response" % (k, d.name, )) - self.assertContains(r, d.title, msg_prefix="%s '%s' title not in response" % (k, d.title, )) - + self.assertContains(r, d.title, msg_prefix="%s '%s' not in response" % (k, d.title, )) + def test_past_documents(self): url = urlreverse("ietf.iesg.views.past_documents") # We haven't put any documents on past telechats, so this should be empty @@ -589,6 +589,66 @@ def test_admin_change(self): draft = Document.objects.get(name="draft-ietf-mars-test") self.assertEqual(draft.telechat_date(),today) +class IESGAgendaTelechatPagesTests(TestCase): + def setUp(self): + super().setUp() + # make_immutable_test_data made a set of future telechats - only need one + # We'll take the "next" one + self.telechat_date = get_agenda_date() + # make_immutable_test_data made and area with only one ad - give it another + ad = Person.objects.get(user__username="ad") + adrole = Role.objects.get(person=ad, name="ad") + ad2 = RoleFactory(group=adrole.group, name_id="ad").person + self.ads=[ad,ad2] + + # Make some drafts + docs = [ + WgDraftFactory(pages=2, states=[('draft-iesg','iesg-eva'),]), + IndividualDraftFactory(pages=20, states=[('draft-iesg','iesg-eva'),]), + WgDraftFactory(pages=200, states=[('draft-iesg','iesg-eva'),]), + ] + # Put them on the telechat + for doc in docs: + TelechatDocEventFactory(doc=doc, telechat_date=self.telechat_date) + # Give them ballots + ballots = [BallotDocEventFactory(doc=doc) for doc in docs] + + # Give the "ad" Area-Director a discuss on one + BallotPositionDocEventFactory(balloter=ad, doc=docs[0], pos_id="discuss", ballot=ballots[0]) + # and a "norecord" position on another + BallotPositionDocEventFactory(balloter=ad, doc=docs[1], pos_id="norecord", ballot=ballots[1]) + # Now "ad" should have 220 pages left to ballot on. + # Every other ad should have 222 pages left to ballot on. + + def test_ad_pages_left_to_ballot_on(self): + url = urlreverse("ietf.iesg.views.agenda_documents") + + # A non-AD user won't get "pages left" + response = self.client.get(url) + telechat = response.context["telechats"][0] + self.assertEqual(telechat["date"], self.telechat_date) + self.assertEqual(telechat["ad_pages_left_to_ballot_on"],0) + self.assertNotContains(response,"pages left to ballot on") + + username=self.ads[0].user.username + self.assertTrue(self.client.login(username=username, password=f"{username}+password")) + + response = self.client.get(url) + telechat = response.context["telechats"][0] + self.assertEqual(telechat["ad_pages_left_to_ballot_on"],220) + self.assertContains(response,"220 pages left to ballot on") + + self.client.logout() + username=self.ads[1].user.username + self.assertTrue(self.client.login(username=username, password=f"{username}+password")) + + response = self.client.get(url) + telechat = response.context["telechats"][0] + self.assertEqual(telechat["ad_pages_left_to_ballot_on"],222) + + + + class RescheduleOnAgendaTests(TestCase): def test_reschedule(self): draft = WgDraftFactory() diff --git a/ietf/iesg/utils.py b/ietf/iesg/utils.py index 3f4883798f..a56fa72cee 100644 --- a/ietf/iesg/utils.py +++ b/ietf/iesg/utils.py @@ -7,11 +7,11 @@ from ietf.iesg.agenda import get_doc_section -TelechatPageCount = namedtuple('TelechatPageCount',['for_approval','for_action','related']) +TelechatPageCount = namedtuple('TelechatPageCount',['for_approval','for_action','related','ad_pages_left_to_ballot_on']) -def telechat_page_count(date=None, docs=None): +def telechat_page_count(date=None, docs=None, ad=None): if not date and not docs: - return TelechatPageCount(0, 0, 0) + return TelechatPageCount(0, 0, 0, 0) if not docs: candidates = Document.objects.filter(docevent__telechatdocevent__telechat_date=date).distinct() @@ -24,7 +24,18 @@ def telechat_page_count(date=None, docs=None): drafts = [d for d in for_approval if d.type_id == 'draft'] - pages_for_approval = sum([d.pages or 0 for d in drafts]) + ad_pages_left_to_ballot_on = 0 + pages_for_approval = 0 + + for draft in drafts: + pages_for_approval += draft.pages or 0 + if ad: + ballot = draft.active_ballot() + if ballot: + positions = ballot.active_balloter_positions() + ad_position = positions[ad] + if ad_position is None or ad_position.pos_id == "norecord": + ad_pages_left_to_ballot_on += draft.pages or 0 pages_for_action = 0 for d in for_action: @@ -53,4 +64,5 @@ def telechat_page_count(date=None, docs=None): return TelechatPageCount(for_approval=pages_for_approval, for_action=pages_for_action, - related=related_pages) + related=related_pages, + ad_pages_left_to_ballot_on=ad_pages_left_to_ballot_on) diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index a92d617ac5..df02754f2e 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -360,6 +360,8 @@ def handle_reschedule_form(request, doc, dates, status): return form def agenda_documents(request): + ad = request.user.person if has_role(request.user, "Area Director") else None + dates = list(TelechatDate.objects.active().order_by('date').values_list("date", flat=True)[:4]) docs_by_date = dict((d, []) for d in dates) @@ -389,11 +391,13 @@ def agenda_documents(request): # the search_result_row view to display them (which expects them) fill_in_document_table_attributes(docs_by_date[date], have_telechat_date=True) fill_in_agenda_docs(date, sections, docs_by_date[date]) - pages = telechat_page_count(docs=docs_by_date[date]).for_approval - + page_count = telechat_page_count(docs=docs_by_date[date], ad=ad) + pages = page_count.for_approval + telechats.append({ "date": date, "pages": pages, + "ad_pages_left_to_ballot_on": page_count.ad_pages_left_to_ballot_on, "sections": sorted((num, section) for num, section in sections.items() if "2" <= num < "5") }) diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html index cfc8830e50..a73264c0f3 100644 --- a/ietf/templates/doc/ad_list.html +++ b/ietf/templates/doc/ad_list.html @@ -35,9 +35,12 @@

{{ dt.type.1 }} State Counts

Area Director + {% if dt.type.1 == "Internet-Draft" %} + Pre pubreq + {% endif %} {% for state, state_name in dt.states %} - + {{ state_name|split:'/'|join:'/' }} @@ -51,6 +54,17 @@

{{ dt.type.1 }} State Counts

{{ ad.name }} + {% if dt.type.1 == "Internet-Draft" %} + + {{ ad.pre_pubreq }} + + {% endif %} {% for state, state_name in dt.states %} @@ -63,6 +77,16 @@

{{ dt.type.1 }} State Counts

Sum + {% if dt.type.1 == "Internet-Draft" %} + +
+ + {% endif %} {% for state, state_name in dt.states %}
@@ -87,37 +111,151 @@

{{ dt.type.1 }} State Counts

{{ data|json_script:"data" }} + + + + {% endblock %} \ No newline at end of file diff --git a/ietf/templates/iesg/agenda_documents.html b/ietf/templates/iesg/agenda_documents.html index 80dd9956fa..f732672df0 100644 --- a/ietf/templates/iesg/agenda_documents.html +++ b/ietf/templates/iesg/agenda_documents.html @@ -21,7 +21,12 @@

Documents on future IESG telechat agendas

IESG telechat {{ t.date }}
- {{ t.pages }} page{{ t.pages|pluralize }} + + {{ t.pages }} page{{ t.pages|pluralize }} + {% if t.ad_pages_left_to_ballot_on %} + ({{ t.ad_pages_left_to_ballot_on }} pages left to ballot on) + {% endif %} +

diff --git a/playwright/tests-legacy/docs/ad.spec.js b/playwright/tests-legacy/docs/ad.spec.js new file mode 100644 index 0000000000..80b8b27cda --- /dev/null +++ b/playwright/tests-legacy/docs/ad.spec.js @@ -0,0 +1,26 @@ +const { test, expect } = require('@playwright/test') +const viewports = require('../../helpers/viewports') + +// ==================================================================== +// IESG Dashboard +// ==================================================================== + +test.describe('/doc/ad/', () => { + test.beforeEach(async ({ page }) => { + await page.setViewportSize({ + width: viewports.desktop[0], + height: viewports.desktop[1] + }) + + await page.goto('/doc/ad/') + }) + + test('Pre pubreq', async ({ page }) => { + const tablesLocator = page.locator('table') + const tablesCount = await tablesLocator.count() + expect(tablesCount).toBeGreaterThan(0) + const firstTable = tablesLocator.nth(0) + const theadTexts = await firstTable.locator('thead').allInnerTexts() + expect(theadTexts.join('')).toContain('Pre pubreq') + }) +}) From d8d52eedbf6508d219fbabbfb0e9f36bc6ef0160 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 9 Sep 2024 12:00:54 -0300 Subject: [PATCH 007/601] feat: email ingestor api test endpoint (#7915) * feat: email ingestor api test endpoint * ci: add ingestion test token for sandbox * chore: fix comments --- dev/deploy-to-container/settings_local.py | 6 ++ ietf/api/tests.py | 97 ++++++++++++++++++++++- ietf/api/urls.py | 2 + ietf/api/views.py | 41 ++++++++-- 4 files changed, 138 insertions(+), 8 deletions(-) diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index 25eacc3004..ae698e20b6 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -71,5 +71,11 @@ DE_GFM_BINARY = '/usr/local/bin/de-gfm' +# No real secrets here, these are public testing values _only_ +APP_API_TOKENS = { + "ietf.api.views.ingest_email_test": ["ingestion-test-token"] +} + + # OIDC configuration SITE_URL = 'https://__HOSTNAME__' diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 20c3e2cb44..4f2a7f7d3c 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -1022,7 +1022,9 @@ def test_role_holder_addresses(self): sorted(e.address for e in emails), ) - @override_settings(APP_API_TOKENS={"ietf.api.views.ingest_email": "valid-token"}) + @override_settings( + APP_API_TOKENS={"ietf.api.views.ingest_email": "valid-token", "ietf.api.views.ingest_email_test": "test-token"} + ) @mock.patch("ietf.api.views.iana_ingest_review_email") @mock.patch("ietf.api.views.ipr_ingest_response_email") @mock.patch("ietf.api.views.nomcom_ingest_feedback_email") @@ -1032,29 +1034,47 @@ def test_ingest_email( mocks = {mock_nomcom_ingest, mock_ipr_ingest, mock_iana_ingest} empty_outbox() url = urlreverse("ietf.api.views.ingest_email") + test_mode_url = urlreverse("ietf.api.views.ingest_email_test") # test various bad calls r = self.client.get(url) self.assertEqual(r.status_code, 403) self.assertFalse(any(m.called for m in mocks)) + r = self.client.get(test_mode_url) + self.assertEqual(r.status_code, 403) + self.assertFalse(any(m.called for m in mocks)) r = self.client.post(url) self.assertEqual(r.status_code, 403) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post(test_mode_url) + self.assertEqual(r.status_code, 403) + self.assertFalse(any(m.called for m in mocks)) r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) self.assertEqual(r.status_code, 405) self.assertFalse(any(m.called for m in mocks)) + r = self.client.get(test_mode_url, headers={"X-Api-Key": "test-token"}) + self.assertEqual(r.status_code, 405) + self.assertFalse(any(m.called for m in mocks)) r = self.client.post(url, headers={"X-Api-Key": "valid-token"}) self.assertEqual(r.status_code, 415) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post(test_mode_url, headers={"X-Api-Key": "test-token"}) + self.assertEqual(r.status_code, 415) + self.assertFalse(any(m.called for m in mocks)) r = self.client.post( url, content_type="application/json", headers={"X-Api-Key": "valid-token"} ) self.assertEqual(r.status_code, 400) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post( + test_mode_url, content_type="application/json", headers={"X-Api-Key": "test-token"} + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(any(m.called for m in mocks)) r = self.client.post( url, @@ -1064,6 +1084,14 @@ def test_ingest_email( ) self.assertEqual(r.status_code, 400) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post( + test_mode_url, + "this is not JSON!", + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(any(m.called for m in mocks)) r = self.client.post( url, @@ -1073,6 +1101,14 @@ def test_ingest_email( ) self.assertEqual(r.status_code, 400) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post( + test_mode_url, + {"json": "yes", "valid_schema": False}, + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(any(m.called for m in mocks)) # bad destination message_b64 = base64.b64encode(b"This is a message").decode() @@ -1086,6 +1122,16 @@ def test_ingest_email( self.assertEqual(r.headers["Content-Type"], "application/json") self.assertEqual(json.loads(r.content), {"result": "bad_dest"}) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post( + test_mode_url, + {"dest": "not-a-destination", "message": message_b64}, + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + self.assertEqual(json.loads(r.content), {"result": "bad_dest"}) + self.assertFalse(any(m.called for m in mocks)) # test that valid requests call handlers appropriately r = self.client.post( @@ -1102,6 +1148,19 @@ def test_ingest_email( self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest}))) mock_iana_ingest.reset_mock() + # the test mode endpoint should _not_ call the handler + r = self.client.post( + test_mode_url, + {"dest": "iana-review", "message": message_b64}, + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + self.assertEqual(json.loads(r.content), {"result": "ok"}) + self.assertFalse(any(m.called for m in mocks)) + mock_iana_ingest.reset_mock() + r = self.client.post( url, {"dest": "ipr-response", "message": message_b64}, @@ -1116,6 +1175,19 @@ def test_ingest_email( self.assertFalse(any(m.called for m in (mocks - {mock_ipr_ingest}))) mock_ipr_ingest.reset_mock() + # the test mode endpoint should _not_ call the handler + r = self.client.post( + test_mode_url, + {"dest": "ipr-response", "message": message_b64}, + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + self.assertEqual(json.loads(r.content), {"result": "ok"}) + self.assertFalse(any(m.called for m in mocks)) + mock_ipr_ingest.reset_mock() + # bad nomcom-feedback dest for bad_nomcom_dest in [ "nomcom-feedback", # no suffix @@ -1133,6 +1205,16 @@ def test_ingest_email( self.assertEqual(r.headers["Content-Type"], "application/json") self.assertEqual(json.loads(r.content), {"result": "bad_dest"}) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post( + test_mode_url, + {"dest": bad_nomcom_dest, "message": message_b64}, + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + self.assertEqual(json.loads(r.content), {"result": "bad_dest"}) + self.assertFalse(any(m.called for m in mocks)) # good nomcom-feedback dest random_year = randrange(100000) @@ -1150,6 +1232,19 @@ def test_ingest_email( self.assertFalse(any(m.called for m in (mocks - {mock_nomcom_ingest}))) mock_nomcom_ingest.reset_mock() + # the test mode endpoint should _not_ call the handler + r = self.client.post( + test_mode_url, + {"dest": f"nomcom-feedback-{random_year}", "message": message_b64}, + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + self.assertEqual(json.loads(r.content), {"result": "ok"}) + self.assertFalse(any(m.called for m in mocks)) + mock_nomcom_ingest.reset_mock() + # test that exceptions lead to email being sent - assumes that iana-review handling is representative mock_iana_ingest.side_effect = EmailIngestionError("Error: don't send email") r = self.client.post( diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 3c0fb872c9..396b3813d6 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -27,6 +27,8 @@ url(r'^doc/draft-aliases/$', api_views.draft_aliases), # email ingestor url(r'email/$', api_views.ingest_email), + # email ingestor + url(r'email/test/$', api_views.ingest_email_test), # GDPR: export of personal information for the logged-in person url(r'^export/personal-information/$', api_views.PersonalInformationExportView.as_view()), # Email alias information for groups diff --git a/ietf/api/views.py b/ietf/api/views.py index 62857bff54..f8662f9a0e 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -614,14 +614,16 @@ def as_emailmessage(self) -> Optional[EmailMessage]: return msg -@requires_api_token -@csrf_exempt -def ingest_email(request): - """Ingest incoming email +def ingest_email_handler(request, test_mode=False): + """Ingest incoming email - handler Returns a 4xx or 5xx status code if the HTTP request was invalid or something went wrong while processing it. If the request was valid, returns a 200. This may or may not indicate that the message was accepted. + + If test_mode is true, actual processing of a valid message will be skipped. In this + mode, a valid request with a valid destination will be treated as accepted. The + "bad_dest" error may still be returned. """ def _http_err(code, text): @@ -657,15 +659,18 @@ def _api_response(result): try: if dest == "iana-review": valid_dest = True - iana_ingest_review_email(message) + if not test_mode: + iana_ingest_review_email(message) elif dest == "ipr-response": valid_dest = True - ipr_ingest_response_email(message) + if not test_mode: + ipr_ingest_response_email(message) elif dest.startswith("nomcom-feedback-"): maybe_year = dest[len("nomcom-feedback-"):] if maybe_year.isdecimal(): valid_dest = True - nomcom_ingest_feedback_email(message, int(maybe_year)) + if not test_mode: + nomcom_ingest_feedback_email(message, int(maybe_year)) except EmailIngestionError as err: error_email = err.as_emailmessage() if error_email is not None: @@ -677,3 +682,25 @@ def _api_response(result): return _api_response("bad_dest") return _api_response("ok") + + +@requires_api_token +@csrf_exempt +def ingest_email(request): + """Ingest incoming email + + Hands off to ingest_email_handler() with test_mode=False. This allows @requires_api_token to + give the test endpoint a distinct token from the real one. + """ + return ingest_email_handler(request, test_mode=False) + + +@requires_api_token +@csrf_exempt +def ingest_email_test(request): + """Ingest incoming email test endpoint + + Hands off to ingest_email_handler() with test_mode=True. This allows @requires_api_token to + give the test endpoint a distinct token from the real one. + """ + return ingest_email_handler(request, test_mode=True) From 80599f29f0dacb9069f2963aa0714f91889d10db Mon Sep 17 00:00:00 2001 From: Sangho Na Date: Wed, 11 Sep 2024 10:22:45 +1200 Subject: [PATCH 008/601] fix: Include missing related drafts in IPR searches (#7836) * fix: Include missing related drafts in IPR searches * refactor: extract drafts, sort docs * chore: indent loop and conditionals to improve readability * test: handle whitespaces added to IPR search result page --------- Co-authored-by: Robert Sparks --- ietf/ipr/tests.py | 33 ++++++++++++++++++++ ietf/ipr/views.py | 38 ++++++++++++++++++++--- ietf/templates/ipr/search_doc_result.html | 19 +++++++++--- 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py index 922ae272a9..a65b0d6c6c 100644 --- a/ietf/ipr/tests.py +++ b/ietf/ipr/tests.py @@ -24,6 +24,7 @@ RfcFactory, NewRevisionDocEventFactory ) +from ietf.doc.utils import prettify_std_name from ietf.group.factories import RoleFactory from ietf.ipr.factories import ( HolderIprDisclosureFactory, @@ -192,6 +193,38 @@ def test_search(self): r = self.client.get(url + "?submit=rfc&rfc=321") self.assertContains(r, ipr.title) + rfc_new = RfcFactory(rfc_number=322) + rfc_new.relateddocument_set.create(relationship_id="obs", target=rfc) + + # find RFC 322 which obsoletes RFC 321 whose draft has IPR + r = self.client.get(url + "?submit=rfc&rfc=322") + self.assertContains(r, ipr.title) + self.assertContains(r, "Total number of IPR disclosures found: 1") + self.assertContains(r, "Total number of documents searched: 3.") + self.assertContains( + r, + f"""Results for {prettify_std_name(rfc_new.name)} + ("{rfc_new.title}")""", + ) + self.assertContains( + r, + f"""Results for {prettify_std_name(rfc.name)} + ("{rfc.title}"), which + + was obsoleted by + {prettify_std_name(rfc_new.name)} + ("{rfc_new.title}")""", + ) + self.assertContains( + r, + f"""Results for {prettify_std_name(draft.name)} + ("{draft.title}"), which + + became rfc + {prettify_std_name(rfc.name)} + ("{rfc.title}")""", + ) + # find by patent owner r = self.client.get(url + "?submit=holder&holder=%s" % ipr.holder_legal_name) self.assertContains(r, ipr.title) diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py index a061232b8f..45fad9a2cc 100644 --- a/ietf/ipr/views.py +++ b/ietf/ipr/views.py @@ -689,11 +689,41 @@ def search(request): if len(start) == 1: first = start[0] doc = first - docs = related_docs(first) - iprs = iprs_from_docs(docs,states=states) + docs = set([first]) + docs.update( + related_docs( + first, relationship=("replaces", "obs"), reverse_relationship=() + ) + ) + docs.update( + set( + [ + draft + for drafts in [ + related_docs( + d, relationship=(), reverse_relationship=("became_rfc",) + ) + for d in docs + ] + for draft in drafts + ] + ) + ) + docs.discard(None) + docs = sorted( + docs, + key=lambda d: ( + d.rfc_number if d.rfc_number is not None else 0, + d.became_rfc().rfc_number if d.became_rfc() else 0, + ), + reverse=True, + ) + iprs = iprs_from_docs(docs, states=states) template = "ipr/search_doc_result.html" - updated_docs = related_docs(first, ('updates',)) - related_iprs = list(set(iprs_from_docs(updated_docs, states=states)) - set(iprs)) + updated_docs = related_docs(first, ("updates",)) + related_iprs = list( + set(iprs_from_docs(updated_docs, states=states)) - set(iprs) + ) # multiple matches, select just one elif start: docs = start diff --git a/ietf/templates/ipr/search_doc_result.html b/ietf/templates/ipr/search_doc_result.html index dc7d8b95b5..b003a32108 100644 --- a/ietf/templates/ipr/search_doc_result.html +++ b/ietf/templates/ipr/search_doc_result.html @@ -54,16 +54,27 @@ - {% for doc in docs %} + {% for d in docs %} - Results for {{ doc.name|prettystdname|urlize_ietf_docs }} ("{{ doc.title }}"){% if not forloop.first %}{% if doc.related %}, which was {{ doc.relation|lower }} {{ doc.related.source|prettystdname|urlize_ietf_docs }} ("{{ doc.related.source.title }}"){% endif %}{% endif %} + Results for {{ d.name|prettystdname|urlize_ietf_docs }} + ("{{ d.title }}"){% if d != doc and d.related %}, which + {% if d == d.related.source %} + {{ d.relation|lower }} + {{ d.related.target|prettystdname|urlize_ietf_docs }} + ("{{ d.related.target.title }}") + {% else %} + was {{ d.relation|lower }} + {{ d.related.source|prettystdname|urlize_ietf_docs }} + ("{{ d.related.source.title }}") + {% endif %} + {% endif %} - {% with doc.iprdocrel_set.all as doc_iprs %} + {% with d.iprdocrel_set.all as doc_iprs %} {% if doc_iprs %} {% for ipr in doc_iprs %} {% if ipr.disclosure.state_id in states %} @@ -81,7 +92,7 @@ - No IPR disclosures have been submitted directly on {{ doc.name|prettystdname|urlize_ietf_docs }}{% if iprs %}, + No IPR disclosures have been submitted directly on {{ d.name|prettystdname|urlize_ietf_docs }}{% if iprs %}, but there are disclosures on {% if docs|length == 2 %}a related document{% else %}related documents{% endif %}, listed on this page{% endif %}. From 13aa072a1e6f50f40a3b33a823408f8eae3b6914 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 12 Sep 2024 13:46:03 -0300 Subject: [PATCH 009/601] chore(deps): pin importlib-metadata (#7927) --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 88b78e665b..c51ed9ac00 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,7 @@ gunicorn>=20.1.0 hashids>=1.3.1 html2text>=2020.1.16 # Used only to clean comment field of secr/sreq html5lib>=1.1 # Only used in tests +importlib-metadata<8.5.0 # indirect req of Markdown/inflect; https://github.com/ietf-tools/datatracker/issues/7924 inflect>= 6.0.2 jsonfield>=3.1.0 # for SubmissionCheck. This is https://github.com/bradjasper/django-jsonfield/. jsonschema[format]>=4.2.1 From 8d608a1282bf84a8b5df0a9d12f16b25cf265f4b Mon Sep 17 00:00:00 2001 From: Sangho Na Date: Fri, 13 Sep 2024 05:47:15 +1200 Subject: [PATCH 010/601] test: check HTML content with whitespace ignored (#7921) --- ietf/ipr/tests.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py index a65b0d6c6c..b08e359462 100644 --- a/ietf/ipr/tests.py +++ b/ietf/ipr/tests.py @@ -203,26 +203,20 @@ def test_search(self): self.assertContains(r, "Total number of documents searched: 3.") self.assertContains( r, - f"""Results for {prettify_std_name(rfc_new.name)} - ("{rfc_new.title}")""", + f'Results for {prettify_std_name(rfc_new.name)} ("{rfc_new.title}")', + html=True, ) self.assertContains( r, - f"""Results for {prettify_std_name(rfc.name)} - ("{rfc.title}"), which - - was obsoleted by - {prettify_std_name(rfc_new.name)} - ("{rfc_new.title}")""", + f'Results for {prettify_std_name(rfc.name)} ("{rfc.title}"), ' + f'which was obsoleted by {prettify_std_name(rfc_new.name)} ("{rfc_new.title}")', + html=True, ) self.assertContains( r, - f"""Results for {prettify_std_name(draft.name)} - ("{draft.title}"), which - - became rfc - {prettify_std_name(rfc.name)} - ("{rfc.title}")""", + f'Results for {prettify_std_name(draft.name)} ("{draft.title}"), ' + f'which became rfc {prettify_std_name(rfc.name)} ("{rfc.title}")', + html=True, ) # find by patent owner From f0f2b6d8b6a7a44f85047c42fefe89c8ca586f09 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Fri, 13 Sep 2024 05:47:53 +1200 Subject: [PATCH 011/601] test: Use timezone aware datetime (#7918) * test: Use timezone aware datetime * fixup! test: Use timezone aware datetime --- ietf/meeting/tests_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py index 0a089ee9e8..f2949a7bb5 100644 --- a/ietf/meeting/tests_models.py +++ b/ietf/meeting/tests_models.py @@ -158,7 +158,7 @@ def test_alpha_str(self): def test_session_recording_url(self): group_acronym = "foobar" - meeting_date = datetime.date.today() + meeting_date = date_today() meeting_number = 123 # IETF meeting From b8c6cb34dd335de88740563eab6ebfbd362b49f4 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Fri, 13 Sep 2024 09:28:44 +1200 Subject: [PATCH 012/601] chore: Remove obsolete `version` attribute (#7931) --- docker-compose.yml | 2 -- docker/docker-compose.celery.yml | 4 ---- docker/docker-compose.extend.yml | 2 -- 3 files changed, 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 413c04ff63..65b28f54fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: app: build: diff --git a/docker/docker-compose.celery.yml b/docker/docker-compose.celery.yml index dedae2d004..b6cc3d09e8 100644 --- a/docker/docker-compose.celery.yml +++ b/docker/docker-compose.celery.yml @@ -1,7 +1,3 @@ -version: '2.4' -# Use version 2.4 for mem_limit setting. Version 3+ uses deploy.resources.limits.memory -# instead, but that only works for swarm with docker-compose 1.25.1. - services: mq: image: rabbitmq:3-alpine diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml index d055c976f4..0538c0d3e9 100644 --- a/docker/docker-compose.extend.yml +++ b/docker/docker-compose.extend.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: app: ports: From 65547a7a9dfa51d60ab31b561590ff4b1aee7033 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 12 Sep 2024 17:04:12 -0500 Subject: [PATCH 013/601] fix: rectify mixed types in gathering mailtrigger recipients (#7932) --- ietf/mailtrigger/models.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ietf/mailtrigger/models.py b/ietf/mailtrigger/models.py index aea530083a..66b7139fa5 100644 --- a/ietf/mailtrigger/models.py +++ b/ietf/mailtrigger/models.py @@ -138,16 +138,16 @@ def gather_group_steering_group(self,**kwargs): def gather_stream_managers(self, **kwargs): addrs = [] manager_map = dict( - ise = '', - irtf = '', - ietf = '', - iab = '', + ise = [''], + irtf = [''], + ietf = [''], + iab = [''], editorial = Role.objects.filter(group__acronym="rsab",name_id="chair").values_list("email__address", flat=True), ) if 'streams' in kwargs: for stream in kwargs['streams']: if stream in manager_map: - addrs.append(manager_map[stream]) + addrs.extend(manager_map[stream]) return addrs def gather_doc_stream_manager(self, **kwargs): @@ -234,7 +234,7 @@ def gather_submission_submitter(self, **kwargs): try: submitter = Alias.objects.get(name=submission.submitter).person if submitter and submitter.email(): - addrs.extend(["%s <%s>" % (submitter.name, submitter.email().address)]) + addrs.append(f"{submitter.name} <{submitter.email().address}>") except (Alias.DoesNotExist, Alias.MultipleObjectsReturned): pass return addrs From f5c132a20a695915e288195b32191b318bd24b9b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 13 Sep 2024 18:42:51 -0300 Subject: [PATCH 014/601] refactor: helper for session recording URL; fix test (#7933) * refactor: session recording URL label helper * test: update tests, avoid tz dependence * test: use date_today() --- ietf/meeting/models.py | 14 ++++---- ietf/meeting/tests_models.py | 67 +++++++++++++++++------------------- 2 files changed, 39 insertions(+), 42 deletions(-) diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 01b695bcea..e169077800 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -1331,16 +1331,18 @@ def onsite_tool_url(self): return url.format(session=self) return None + def _session_recording_url_label(self): + if self.meeting.type.slug == "ietf" and self.has_onsite_tool: + session_label = f"IETF{self.meeting.number}-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}" + else: + session_label = f"IETF-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}" + return session_label + def session_recording_url(self): url_formatter = getattr(settings, "MEETECHO_SESSION_RECORDING_URL", "") url = None if url_formatter and self.video_stream_url: - if self.meeting.type.slug == "ietf" and self.has_onsite_tool: - session_label = f"IETF{self.meeting.number}-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}" - else: - session_label = f"IETF-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}" - url = url_formatter.format(session_label=session_label) - + url = url_formatter.format(session_label=self._session_recording_url_label()) return url diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py index f2949a7bb5..8457423c51 100644 --- a/ietf/meeting/tests_models.py +++ b/ietf/meeting/tests_models.py @@ -8,6 +8,7 @@ from django.conf import settings from django.test import override_settings +import ietf.meeting.models from ietf.group.factories import GroupFactory, GroupHistoryFactory from ietf.meeting.factories import MeetingFactory, SessionFactory, AttendedFactory, SessionPresentationFactory from ietf.meeting.models import Session @@ -156,46 +157,40 @@ def test_alpha_str(self): self.assertEqual(Session._alpha_str(27 * 26 - 1), "zz") self.assertEqual(Session._alpha_str(27 * 26), "aaa") - def test_session_recording_url(self): - group_acronym = "foobar" - meeting_date = date_today() - meeting_number = 123 - - # IETF meeting + @patch.object(ietf.meeting.models.Session, "_session_recording_url_label", return_value="LABEL") + def test_session_recording_url(self, mock): + for session_type in ["ietf", "interim"]: + session = SessionFactory(meeting__type_id=session_type) + with override_settings(): + if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"): + del settings.MEETECHO_SESSION_RECORDING_URL + self.assertIsNone(session.session_recording_url()) + + settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com" + self.assertEqual(session.session_recording_url(), "http://player.example.com") + + settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}" + self.assertEqual(session.session_recording_url(), "http://player.example.com?LABEL") + + def test_session_recording_url_label_ietf(self): session = SessionFactory( meeting__type_id='ietf', - meeting__date=meeting_date, - group__acronym=group_acronym, - meeting__number=meeting_number, + meeting__date=date_today(), + meeting__number="123", + group__acronym="acro", ) - with override_settings(): - if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"): - del settings.MEETECHO_SESSION_RECORDING_URL - self.assertIsNone(session.session_recording_url()) - - settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com" - self.assertEqual(session.session_recording_url(), "http://player.example.com") - - settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}" - self.assertIn(f"IETF{meeting_number}-{group_acronym.upper()}", session.session_recording_url()) - self.assertIn(f"{meeting_date.strftime('%Y%m%d')}", session.session_recording_url()) - self.assertTrue(session.session_recording_url().startswith("http://player.example.com")) + session_time = session.official_timeslotassignment().timeslot.time + self.assertEqual( + f"IETF123-ACRO-{session_time:%Y%m%d-%H%M}", # n.b., time in label is UTC + session._session_recording_url_label()) - # interim meeting + def test_session_recording_url_label_interim(self): session = SessionFactory( meeting__type_id='interim', - meeting__date=meeting_date, - group__acronym=group_acronym, + meeting__date=date_today(), + group__acronym="acro", ) - with override_settings(): - if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"): - del settings.MEETECHO_SESSION_RECORDING_URL - self.assertIsNone(session.session_recording_url()) - - settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com" - self.assertEqual(session.session_recording_url(), "http://player.example.com") - - settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}" - self.assertIn(f"IETF-{group_acronym.upper()}", session.session_recording_url()) - self.assertIn(f"{meeting_date.strftime('%Y%m%d')}", session.session_recording_url()) - self.assertTrue(session.session_recording_url().startswith("http://player.example.com")) + session_time = session.official_timeslotassignment().timeslot.time + self.assertEqual( + f"IETF-ACRO-{session_time:%Y%m%d-%H%M}", # n.b., time in label is UTC + session._session_recording_url_label()) From 3b5058a51693c3bb4bf526a089f08c906f584499 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 16 Sep 2024 08:56:51 -0500 Subject: [PATCH 015/601] fix: start to reconcile internal inconsistencies wrt multiple from values (#7935) --- ietf/utils/mail.py | 14 +++++++++++++- ietf/utils/tests.py | 12 ++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/ietf/utils/mail.py b/ietf/utils/mail.py index e747c74778..4585fdb846 100644 --- a/ietf/utils/mail.py +++ b/ietf/utils/mail.py @@ -92,7 +92,17 @@ def send_smtp(msg, bcc=None): ''' mark = time.time() add_headers(msg) - (fname, frm) = parseaddr(msg.get('From')) + # N.B. We have a disconnect with most of this code assuming a From header value will only + # have one address. + # The frm computed here is only used as the envelope from. + # Previous code simply ran `parseaddr(msg.get('From'))`, getting lucky if the string returned + # from the get had more than one address in it. Python 3.9.20 changes the behavior of parseaddr + # and that erroneous use of the function no longer gets lucky. + # For the short term, to match behavior to date as closely as possible, if we get a message + # that has multiple addresses in the From header, we will use the first for the envelope from + from_tuples = getaddresses(msg.get_all('From', [settings.DEFAULT_FROM_EMAIL])) + assertion('len(from_tuples)==1', note=f"send_smtp received multiple From addresses: {from_tuples}") + _ , frm = from_tuples[0] addrlist = msg.get_all('To') + msg.get_all('Cc', []) if bcc: addrlist += [bcc] @@ -446,6 +456,8 @@ def parse_preformatted(preformatted, extra=None, override=None): values = msg.get_all(key, []) if values: values = getaddresses(values) + if key=='From': + assertion('len(values)<2', note=f'parse_preformatted is constructing a From with multiple values: {values}') del msg[key] msg[key] = ',\n '.join(formataddr(v) for v in values) for key in ['Subject', ]: diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index d435583e89..decdd778d9 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -53,7 +53,7 @@ class SendingMail(TestCase): def test_send_mail_preformatted(self): msg = """To: to1@example.com, to2@example.com -From: from1@ietf.org, from2@ietf.org +From: from1@ietf.org Cc: cc1@example.com, cc2@example.com Bcc: bcc1@example.com, bcc2@example.com Subject: subject @@ -63,7 +63,7 @@ def test_send_mail_preformatted(self): send_mail_preformatted(None, msg, {}, {}) recv = outbox[-1] self.assertSameEmail(recv['To'], ', ') - self.assertSameEmail(recv['From'], 'from1@ietf.org, from2@ietf.org') + self.assertSameEmail(recv['From'], 'from1@ietf.org') self.assertSameEmail(recv['Cc'], 'cc1@example.com, cc2@example.com') self.assertSameEmail(recv['Bcc'], None) self.assertEqual(recv['Subject'], 'subject') @@ -71,14 +71,14 @@ def test_send_mail_preformatted(self): override = { 'To': 'oto1@example.net, oto2@example.net', - 'From': 'ofrom1@ietf.org, ofrom2@ietf.org', + 'From': 'ofrom1@ietf.org', 'Cc': 'occ1@example.net, occ2@example.net', 'Subject': 'osubject', } send_mail_preformatted(request=None, preformatted=msg, extra={}, override=override) recv = outbox[-1] self.assertSameEmail(recv['To'], ', ') - self.assertSameEmail(recv['From'], 'ofrom1@ietf.org, ofrom2@ietf.org') + self.assertSameEmail(recv['From'], 'ofrom1@ietf.org') self.assertSameEmail(recv['Cc'], 'occ1@example.net, occ2@example.net') self.assertSameEmail(recv['Bcc'], None) self.assertEqual(recv['Subject'], 'osubject') @@ -86,14 +86,14 @@ def test_send_mail_preformatted(self): override = { 'To': ['', 'oto2@example.net'], - 'From': ['', 'ofrom2@ietf.org'], + 'From': [''], 'Cc': ['', 'occ2@example.net'], 'Subject': 'osubject', } send_mail_preformatted(request=None, preformatted=msg, extra={}, override=override) recv = outbox[-1] self.assertSameEmail(recv['To'], ', ') - self.assertSameEmail(recv['From'], ', ofrom2@ietf.org') + self.assertSameEmail(recv['From'], '') self.assertSameEmail(recv['Cc'], ', occ2@example.net') self.assertSameEmail(recv['Bcc'], None) self.assertEqual(recv['Subject'], 'osubject') From cc1eade4f8b2894adba41d658521b4b0d585e2a3 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 16 Sep 2024 08:57:05 -0500 Subject: [PATCH 016/601] fix: correct headers for charter evaluation email (#7937) --- ietf/templates/doc/charter/issue_ballot_mail.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/templates/doc/charter/issue_ballot_mail.txt b/ietf/templates/doc/charter/issue_ballot_mail.txt index 12fc44bbbc..914935bb12 100644 --- a/ietf/templates/doc/charter/issue_ballot_mail.txt +++ b/ietf/templates/doc/charter/issue_ballot_mail.txt @@ -1,6 +1,6 @@ -{% load ietf_filters %}{% autoescape off %}To: {{ to }} {% if cc %} -Cc: {{ cc }} -{% endif %}From: IESG Secretary +{% load ietf_filters %}{% autoescape off %}To: {{ to }}{% if cc %} +Cc: {{ cc }}{% endif %} +From: IESG Secretary Reply-To: IESG Secretary Subject: Evaluation: {{ doc.name }} From 9d583ab9eb056de5407ec88ba631a07b2d609879 Mon Sep 17 00:00:00 2001 From: Sangho Na Date: Tue, 17 Sep 2024 03:06:16 +1200 Subject: [PATCH 017/601] fix: Use email or name when building community list view (#7203) * fix: Use email or name when building community list view instead of email * test: add test case * chore: remove debug * fix: use name in community list menu when no active email is found --------- Co-authored-by: Jennifer Richards --- ietf/community/tests.py | 13 ++++++++++--- ietf/community/views.py | 1 + ietf/templates/community/list_menu.html | 8 ++++---- ietf/templates/community/view_list.html | 2 +- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/ietf/community/tests.py b/ietf/community/tests.py index d76347b70a..84e4370771 100644 --- a/ietf/community/tests.py +++ b/ietf/community/tests.py @@ -108,10 +108,8 @@ def email_or_name_set(self, person): return [e for e in Email.objects.filter(person=person)] + \ [a for a in Alias.objects.filter(person=person)] - def test_view_list(self): - person = self.complex_person(user__username='plain') + def do_view_list_test(self, person): draft = WgDraftFactory() - # without list for id in self.email_or_name_set(person): url = urlreverse(ietf.community.views.view_list, kwargs={ "email_or_name": id }) @@ -134,6 +132,15 @@ def test_view_list(self): self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'") self.assertContains(r, draft.name) + def test_view_list(self): + person = self.complex_person(user__username='plain') + self.do_view_list_test(person) + + def test_view_list_without_active_email(self): + person = self.complex_person(user__username='plain') + person.email_set.update(active=False) + self.do_view_list_test(person) + def test_manage_personal_list(self): person = self.complex_person(user__username='plain') ad = Person.objects.get(user__username='ad') diff --git a/ietf/community/views.py b/ietf/community/views.py index 4a28a391f0..78b8144d60 100644 --- a/ietf/community/views.py +++ b/ietf/community/views.py @@ -68,6 +68,7 @@ def view_list(request, email_or_name=None): 'meta': meta, 'can_manage_list': can_manage_community_list(request.user, clist), 'subscribed': subscribed, + "email_or_name": email_or_name, }) @login_required diff --git a/ietf/templates/community/list_menu.html b/ietf/templates/community/list_menu.html index 0552c76a45..009d01152d 100644 --- a/ietf/templates/community/list_menu.html +++ b/ietf/templates/community/list_menu.html @@ -3,18 +3,18 @@
+ href="{% if clist.group %}{% url "ietf.community.views.feed" acronym=clist.group.acronym %}{% else %}{% url "ietf.community.views.feed" email_or_name=email_or_name %}{% endif %}"> All changes + href="{% if clist.group %}{% url "ietf.community.views.feed" acronym=clist.group.acronym %}{% else %}{% url "ietf.community.views.feed" email_or_name=email_or_name %}{% endif %}?significant=1"> Significant
{% if clist.pk != None %} + href="{% if clist.group %}{% url "ietf.community.views.subscription" acronym=clist.group.acronym %}{% else %}{% url "ietf.community.views.subscription" email_or_name=email_or_name %}{% endif %}"> {% if subscribed %} Change subscription @@ -24,7 +24,7 @@ {% endif %} + href="{% if clist.group %}{% url "ietf.community.views.export_to_csv" acronym=clist.group.acronym %}{% else %}{% url "ietf.community.views.export_to_csv" email_or_name=email_or_name %}{% endif %}"> Export as CSV
diff --git a/ietf/templates/community/view_list.html b/ietf/templates/community/view_list.html index 7ca9f52748..a543eaf7cf 100644 --- a/ietf/templates/community/view_list.html +++ b/ietf/templates/community/view_list.html @@ -12,7 +12,7 @@

{{ clist.long_name }}

{% bootstrap_messages %} {% if can_manage_list %} + href="{% url "ietf.community.views.manage_list" email_or_name=email_or_name %}"> Manage list From d7be91f784e9fb77193b486fece4e1a0a2f70d9c Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 16 Sep 2024 10:28:13 -0500 Subject: [PATCH 018/601] fix: pin pydyf until weasyprint adjusts for its deprecations (#7945) --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index c51ed9ac00..2e6e2714d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,6 +49,7 @@ oic>=1.3 # Used only by tests Pillow>=9.1.0 psycopg2>=2.9.6 pyang>=2.5.3 +pydyf>0.8.0,<0.10.0 # until weasyprint adjusts for 0.10.0 and later pyflakes>=2.4.0 pyopenssl>=22.0.0 # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency pyquery>=1.4.3 From 8362b45c8e94d5bf3585f27b2e821388d9b9fb14 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 17 Sep 2024 13:05:56 -0500 Subject: [PATCH 019/601] fix: optimize and debug has_role and can_manage_some_groups (#7949) * fix: optimize can_manage_some_groups * fix: improve cache key * refactor: extra_role_qs to kwargs and bugfix to cache key * fix: restrict groupman_role matches to active states * chore: styling, decommenting, black --- ietf/group/utils.py | 21 ++++-- ietf/ietfauth/utils.py | 162 +++++++++++++++++++++++++++++------------ 2 files changed, 131 insertions(+), 52 deletions(-) diff --git a/ietf/group/utils.py b/ietf/group/utils.py index 68b618120b..dcf9d83e6f 100644 --- a/ietf/group/utils.py +++ b/ietf/group/utils.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import datetime +from itertools import chain from pathlib import Path from django.db.models import Q @@ -153,17 +154,23 @@ def can_manage_materials(user, group): def can_manage_session_materials(user, group, session): return has_role(user, 'Secretariat') or (group.has_role(user, group.features.matman_roles) and not session.is_material_submission_cutoff()) -# Maybe this should be cached... def can_manage_some_groups(user): if not user.is_authenticated: return False + authroles = set( + chain.from_iterable( + GroupFeatures.objects.values_list("groupman_authroles", flat=True) + ) + ) + extra_role_qs = dict() for gf in GroupFeatures.objects.all(): - for authrole in gf.groupman_authroles: - if has_role(user, authrole): - return True - if Role.objects.filter(name__in=gf.groupman_roles, group__type_id=gf.type_id, person__user=user).exists(): - return True - return False + extra_role_qs[f"{gf.type_id} groupman roles"] = Q( + name__in=gf.groupman_roles, + group__type_id=gf.type_id, + group__state__in=["active", "bof", "proposed"], + ) + return has_role(user, authroles, extra_role_qs=extra_role_qs) + def can_provide_status_update(user, group): if not group.features.acts_like_wg: diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py index 6fa9cddbcb..d8bd67a4e0 100644 --- a/ietf/ietfauth/utils.py +++ b/ietf/ietfauth/utils.py @@ -38,9 +38,10 @@ def has_role(user, role_names, *args, **kwargs): """Determines whether user has any of the given standard roles given. Role names must be a list or, in case of a single value, a string.""" - if not isinstance(role_names, (list, tuple)): - role_names = [ role_names ] - + extra_role_qs = kwargs.get("extra_role_qs", None) + if not isinstance(role_names, (list, tuple, set)): + role_names = [role_names] + if not user or not user.is_authenticated: return False @@ -48,7 +49,13 @@ def has_role(user, role_names, *args, **kwargs): if not hasattr(user, "roles_check_cache"): user.roles_check_cache = {} - key = frozenset(role_names) + keynames = set(role_names) + if extra_role_qs: + keynames.update(set(extra_role_qs.keys())) + year = kwargs.get("year", None) + if year is not None: + keynames.add(f"nomcomyear{year}") + key = frozenset(keynames) if key not in user.roles_check_cache: try: person = user.person @@ -56,54 +63,119 @@ def has_role(user, role_names, *args, **kwargs): return False role_qs = { - "Area Director": Q(person=person, name__in=("pre-ad", "ad"), group__type="area", group__state="active"), - "Secretariat": Q(person=person, name="secr", group__acronym="secretariat"), - "IAB" : Q(person=person, name="member", group__acronym="iab"), - "IANA": Q(person=person, name="auth", group__acronym="iana"), - "RFC Editor": Q(person=person, name="auth", group__acronym="rpc"), - "ISE" : Q(person=person, name="chair", group__acronym="ise"), - "IAD": Q(person=person, name="admdir", group__acronym="ietf"), - "IETF Chair": Q(person=person, name="chair", group__acronym="ietf"), - "IETF Trust Chair": Q(person=person, name="chair", group__acronym="ietf-trust"), - "IRTF Chair": Q(person=person, name="chair", group__acronym="irtf"), - "RSAB Chair": Q(person=person, name="chair", group__acronym="rsab"), - "IAB Chair": Q(person=person, name="chair", group__acronym="iab"), - "IAB Executive Director": Q(person=person, name="execdir", group__acronym="iab"), - "IAB Group Chair": Q(person=person, name="chair", group__type="iab", group__state="active"), - "IAOC Chair": Q(person=person, name="chair", group__acronym="iaoc"), - "WG Chair": Q(person=person,name="chair", group__type="wg", group__state__in=["active","bof", "proposed"]), - "WG Secretary": Q(person=person,name="secr", group__type="wg", group__state__in=["active","bof", "proposed"]), - "RG Chair": Q(person=person,name="chair", group__type="rg", group__state__in=["active","proposed"]), - "RG Secretary": Q(person=person,name="secr", group__type="rg", group__state__in=["active","proposed"]), - "AG Secretary": Q(person=person,name="secr", group__type="ag", group__state__in=["active"]), - "RAG Secretary": Q(person=person,name="secr", group__type="rag", group__state__in=["active"]), - "Team Chair": Q(person=person,name="chair", group__type="team", group__state="active"), - "Program Lead": Q(person=person,name="lead", group__type="program", group__state="active"), - "Program Secretary": Q(person=person,name="secr", group__type="program", group__state="active"), - "Program Chair": Q(person=person,name="chair", group__type="program", group__state="active"), - "EDWG Chair": Q(person=person, name="chair", group__type="edwg", group__state="active"), - "Nomcom Chair": Q(person=person, name="chair", group__type="nomcom", group__acronym__icontains=kwargs.get('year', '0000')), - "Nomcom Advisor": Q(person=person, name="advisor", group__type="nomcom", group__acronym__icontains=kwargs.get('year', '0000')), - "Nomcom": Q(person=person, group__type="nomcom", group__acronym__icontains=kwargs.get('year', '0000')), - "Liaison Manager": Q(person=person,name="liaiman",group__type="sdo",group__state="active", ), - "Authorized Individual": Q(person=person,name="auth",group__type="sdo",group__state="active", ), - "Recording Manager": Q(person=person,name="recman",group__type="ietf",group__state="active", ), - "Reviewer": Q(person=person, name="reviewer", group__state="active"), - "Review Team Secretary": Q(person=person, name="secr", group__reviewteamsettings__isnull=False,group__state="active", ), - "IRSG Member": (Q(person=person, name="member", group__acronym="irsg") | Q(person=person, name="chair", group__acronym="irtf") | Q(person=person, name="atlarge", group__acronym="irsg")), - "RSAB Member": Q(person=person, name="member", group__acronym="rsab"), - "Robot": Q(person=person, name="robot", group__acronym="secretariat"), - } - - filter_expr = Q(pk__in=[]) # ensure empty set is returned if no other terms are added + "Area Director": Q( + name__in=("pre-ad", "ad"), group__type="area", group__state="active" + ), + "Secretariat": Q(name="secr", group__acronym="secretariat"), + "IAB": Q(name="member", group__acronym="iab"), + "IANA": Q(name="auth", group__acronym="iana"), + "RFC Editor": Q(name="auth", group__acronym="rpc"), + "ISE": Q(name="chair", group__acronym="ise"), + "IAD": Q(name="admdir", group__acronym="ietf"), + "IETF Chair": Q(name="chair", group__acronym="ietf"), + "IETF Trust Chair": Q(name="chair", group__acronym="ietf-trust"), + "IRTF Chair": Q(name="chair", group__acronym="irtf"), + "RSAB Chair": Q(name="chair", group__acronym="rsab"), + "IAB Chair": Q(name="chair", group__acronym="iab"), + "IAB Executive Director": Q(name="execdir", group__acronym="iab"), + "IAB Group Chair": Q( + name="chair", group__type="iab", group__state="active" + ), + "IAOC Chair": Q(name="chair", group__acronym="iaoc"), + "WG Chair": Q( + name="chair", + group__type="wg", + group__state__in=["active", "bof", "proposed"], + ), + "WG Secretary": Q( + name="secr", + group__type="wg", + group__state__in=["active", "bof", "proposed"], + ), + "RG Chair": Q( + name="chair", group__type="rg", group__state__in=["active", "proposed"] + ), + "RG Secretary": Q( + name="secr", group__type="rg", group__state__in=["active", "proposed"] + ), + "AG Secretary": Q( + name="secr", group__type="ag", group__state__in=["active"] + ), + "RAG Secretary": Q( + name="secr", group__type="rag", group__state__in=["active"] + ), + "Team Chair": Q(name="chair", group__type="team", group__state="active"), + "Program Lead": Q( + name="lead", group__type="program", group__state="active" + ), + "Program Secretary": Q( + name="secr", group__type="program", group__state="active" + ), + "Program Chair": Q( + name="chair", group__type="program", group__state="active" + ), + "EDWG Chair": Q(name="chair", group__type="edwg", group__state="active"), + "Nomcom Chair": Q( + name="chair", + group__type="nomcom", + group__acronym__icontains=kwargs.get("year", "0000"), + ), + "Nomcom Advisor": Q( + name="advisor", + group__type="nomcom", + group__acronym__icontains=kwargs.get("year", "0000"), + ), + "Nomcom": Q( + group__type="nomcom", + group__acronym__icontains=kwargs.get("year", "0000"), + ), + "Liaison Manager": Q( + name="liaiman", + group__type="sdo", + group__state="active", + ), + "Authorized Individual": Q( + name="auth", + group__type="sdo", + group__state="active", + ), + "Recording Manager": Q( + name="recman", + group__type="ietf", + group__state="active", + ), + "Reviewer": Q(name="reviewer", group__state="active"), + "Review Team Secretary": Q( + name="secr", + group__reviewteamsettings__isnull=False, + group__state="active", + ), + "IRSG Member": ( + Q(name="member", group__acronym="irsg") + | Q(name="chair", group__acronym="irtf") + | Q(name="atlarge", group__acronym="irsg") + ), + "RSAB Member": Q(name="member", group__acronym="rsab"), + "Robot": Q(name="robot", group__acronym="secretariat"), + } + + filter_expr = Q( + pk__in=[] + ) # ensure empty set is returned if no other terms are added for r in role_names: filter_expr |= role_qs[r] + if extra_role_qs: + for r in extra_role_qs: + filter_expr |= extra_role_qs[r] - user.roles_check_cache[key] = bool(Role.objects.filter(filter_expr).exists()) + user.roles_check_cache[key] = bool( + Role.objects.filter(person=person).filter(filter_expr).exists() + ) return user.roles_check_cache[key] + # convenient decorator def passes_test_decorator(test_func, message): From 97b719505ee162d0c55d57edbdd029f766c27a71 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 17 Sep 2024 13:06:10 -0500 Subject: [PATCH 020/601] test: focus on a helper rather than a whole view (#7951) --- ietf/meeting/tests_views.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index e4f62838de..f951eb682d 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -39,7 +39,7 @@ from ietf.group.models import Group, Role, GroupFeatures from ietf.group.utils import can_manage_group from ietf.person.models import Person, PersonalApiKey -from ietf.meeting.helpers import can_approve_interim_request, can_view_interim_request, preprocess_assignments_for_agenda +from ietf.meeting.helpers import can_approve_interim_request, can_request_interim_meeting, can_view_interim_request, preprocess_assignments_for_agenda from ietf.meeting.helpers import send_interim_approval_request, AgendaKeywordTagger from ietf.meeting.helpers import send_interim_meeting_cancellation_notice, send_interim_session_cancellation_notice from ietf.meeting.helpers import send_interim_minutes_reminder, populate_important_dates, update_important_dates @@ -7334,10 +7334,7 @@ def test_cannot_request_interim(self): for gf in GroupFeatures.objects.filter(has_meetings=True): for role_name in all_role_names - set(gf.groupman_roles): role = RoleFactory(group__type_id=gf.type_id,name_id=role_name) - self.client.login(username=role.person.user.username, password=role.person.user.username+'+password') - r = self.client.get(url) - self.assertEqual(r.status_code, 403) - self.client.logout() + self.assertFalse(can_request_interim_meeting(role.person.user)) def test_appears_on_upcoming(self): url = urlreverse('ietf.meeting.views.upcoming') From c6389ba65fdefdce72661f3ee3d21f7d708da841 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 18 Sep 2024 16:02:00 -0300 Subject: [PATCH 021/601] fix: only send state change notifications once (#7953) * feat: split state_change_event create / save * refactor: avoid double-save in rfceditor.py * feat: only send notifications on event creation * test: update test_notification_signal_receiver() * chore: update comment --- ietf/community/models.py | 5 ++++- ietf/community/tests.py | 27 +++++++++++++++++---------- ietf/doc/utils.py | 25 ++++++++++++++++++++++--- ietf/sync/rfceditor.py | 13 +++++++------ 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/ietf/community/models.py b/ietf/community/models.py index b1295461d6..4d820eb0f1 100644 --- a/ietf/community/models.py +++ b/ietf/community/models.py @@ -104,12 +104,15 @@ def notify_events(sender, instance, **kwargs): if not isinstance(instance, DocEvent): return + if not kwargs.get("created", False): + return # only notify on creation + if instance.doc.type_id != 'draft': return if getattr(instance, "skip_community_list_notification", False): return - + # kludge alert: queuing a celery task in response to a signal can cause unexpected attempts to # start a Celery task during tests. To prevent this, don't queue a celery task if we're running # tests. diff --git a/ietf/community/tests.py b/ietf/community/tests.py index 84e4370771..181e9e0fa6 100644 --- a/ietf/community/tests.py +++ b/ietf/community/tests.py @@ -17,6 +17,7 @@ import ietf.community.views from ietf.group.models import Group from ietf.group.utils import setup_default_community_list_for_group +from ietf.doc.factories import DocumentFactory from ietf.doc.models import State from ietf.doc.utils import add_state_change_event from ietf.person.models import Person, Email, Alias @@ -439,39 +440,45 @@ def test_notification_signal_receiver(self, mock_notify_task): This implicitly tests that notify_events is hooked up to the post_save signal. """ # Arbitrary model that's not a DocEvent - p = PersonFactory() + person = PersonFactory() mock_notify_task.reset_mock() # clear any calls that resulted from the factories # be careful overriding SERVER_MODE - we do it here because the method # under test does not make this call when in "test" mode with override_settings(SERVER_MODE="not-test"): - p.save() + person.save() self.assertFalse(mock_notify_task.delay.called) - d = DocEventFactory() - mock_notify_task.reset_mock() # clear any calls that resulted from the factories + # build a DocEvent that is not yet persisted + doc = DocumentFactory() + d = DocEventFactory.build(by=person, doc=doc) + # mock_notify_task.reset_mock() # clear any calls that resulted from the factories # be careful overriding SERVER_MODE - we do it here because the method # under test does not make this call when in "test" mode with override_settings(SERVER_MODE="not-test"): d.save() - self.assertEqual(mock_notify_task.delay.call_count, 1) + self.assertEqual(mock_notify_task.delay.call_count, 1, "notify_task should be run on creation of DocEvent") self.assertEqual(mock_notify_task.delay.call_args, mock.call(event_id = d.pk)) mock_notify_task.reset_mock() + with override_settings(SERVER_MODE="not-test"): + d.save() + self.assertFalse(mock_notify_task.delay.called, "notify_task should not be run save of on existing DocEvent") + + mock_notify_task.reset_mock() + d = DocEventFactory.build(by=person, doc=doc) d.skip_community_list_notification = True # be careful overriding SERVER_MODE - we do it here because the method # under test does not make this call when in "test" mode with override_settings(SERVER_MODE="not-test"): d.save() - self.assertFalse(mock_notify_task.delay.called) + self.assertFalse(mock_notify_task.delay.called, "notify_task should not run when skip_community_list_notification is set") - del(d.skip_community_list_notification) - d.doc.type_id="rfc" # not "draft" - d.doc.save() + d = DocEventFactory.build(by=person, doc=DocumentFactory(type_id="rfc")) # be careful overriding SERVER_MODE - we do it here because the method # under test does not make this call when in "test" mode with override_settings(SERVER_MODE="not-test"): d.save() - self.assertFalse(mock_notify_task.delay.called) + self.assertFalse(mock_notify_task.delay.called, "notify_task should not run on a document with type 'rfc'") @mock.patch("ietf.utils.mail.send_mail_text") def test_notify_event_to_subscribers(self, mock_send_mail_text): diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 74000e598b..97243a20d6 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -398,8 +398,12 @@ def get_unicode_document_content(key, filename, codec='utf-8', errors='ignore'): def tags_suffix(tags): return ("::" + "::".join(t.name for t in tags)) if tags else "" -def add_state_change_event(doc, by, prev_state, new_state, prev_tags=None, new_tags=None, timestamp=None): - """Add doc event to explain that state change just happened.""" + +def new_state_change_event(doc, by, prev_state, new_state, prev_tags=None, new_tags=None, timestamp=None): + """Create unsaved doc event to explain that state change just happened + + Returns None if no state change occurred. + """ if prev_state and new_state: assert prev_state.type_id == new_state.type_id @@ -419,7 +423,22 @@ def add_state_change_event(doc, by, prev_state, new_state, prev_tags=None, new_t e.desc += " from %s" % (prev_state.name + tags_suffix(prev_tags)) if timestamp: e.time = timestamp - e.save() + return e # not saved! + + +def add_state_change_event(doc, by, prev_state, new_state, prev_tags=None, new_tags=None, timestamp=None): + """Add doc event to explain that state change just happened. + + Returns None if no state change occurred. + + Note: Creating a state change DocEvent will trigger notifications to be sent to people subscribed + to the doc via a CommunityList on its first save(). If you need to adjust the event (say, changing + its desc) before that notification is sent, use new_state_change_event() instead and save the + event after making your changes. + """ + e = new_state_change_event(doc, by, prev_state, new_state, prev_tags, new_tags, timestamp) + if e is not None: + e.save() return e diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py index d3371ea36c..6b3482f10d 100644 --- a/ietf/sync/rfceditor.py +++ b/ietf/sync/rfceditor.py @@ -21,7 +21,7 @@ from ietf.doc.models import ( Document, State, StateType, DocEvent, DocRelationshipName, DocTagName, RelatedDocument, RelatedDocHistory ) from ietf.doc.expire import move_draft_files_to_archive -from ietf.doc.utils import add_state_change_event, prettify_std_name, update_action_holders +from ietf.doc.utils import add_state_change_event, new_state_change_event, prettify_std_name, update_action_holders from ietf.group.models import Group from ietf.ipr.models import IprDocRel from ietf.name.models import StdLevelName, StreamName @@ -202,11 +202,14 @@ def update_drafts_from_queue(drafts): if prev_state != next_state: d.set_state(next_state) - e = add_state_change_event(d, system, prev_state, next_state) + e = new_state_change_event(d, system, prev_state, next_state) # unsaved + if e: + if auth48: + e.desc = re.sub(r"(.*)", "\\1" % auth48, e.desc) + e.save() + events.append(e) if auth48: - e.desc = re.sub(r"(.*)", "\\1" % auth48, e.desc) - e.save() # Create or update the auth48 URL whether or not this is a state expected to have one. d.documenturl_set.update_or_create( tag_id='auth48', # look up existing based on this field @@ -215,8 +218,6 @@ def update_drafts_from_queue(drafts): else: # Remove any existing auth48 URL when an update does not have one. d.documenturl_set.filter(tag_id='auth48').delete() - if e: - events.append(e) changed.add(name) From 35074660dc18cb2ede7c209312dfb2ab32deb740 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 12:08:01 -0700 Subject: [PATCH 022/601] chore: Prevent test suite artifact creation in work directory (#6438) * chore: Prevent test suite artifact creation in work directory Also remove a few other stale test assets while I'm here. * Try and fix CI * Change IDSUBMIT_REPOSITORY_PATH * Make the dir * test: clean up media/nomcom directories * test: de-dup settings_temp_path_overrides * chore: remove debug * refactor: avoid premature import of test_utils * refactor: drop useless lambda wrapper --------- Co-authored-by: Jennifer Richards --- dev/tests/settings_local.py | 4 +- docker/configs/settings_local.py | 4 +- docker/scripts/app-create-dirs.sh | 8 +- ietf/meeting/tests_js.py | 11 +- ietf/person/factories.py | 7 +- ietf/settings_test.py | 24 +- ietf/utils/test_utils.py | 13 +- media/.gitignore | 1 - media/photo/nopictureavailable.jpg | Bin 1618 -> 0 bytes media/photo/profile-default.jpg | Bin 1397 -> 0 bytes test/data/profile-default.jpg | Bin 1397 -> 0 bytes test/data/youtube-discovery.json | 10879 ------------------------- test/data/youtube-playlistid.json | 1 - test/data/youtube-playlistitems.json | 1 - test/lib/.gitignore | 1 - test/lib/README | 9 - test/media/floor/.gitignore | 1 - test/media/photo/.gitignore | 1 - 18 files changed, 33 insertions(+), 10932 deletions(-) delete mode 100644 media/.gitignore delete mode 100644 media/photo/nopictureavailable.jpg delete mode 100644 media/photo/profile-default.jpg delete mode 100644 test/data/profile-default.jpg delete mode 100644 test/data/youtube-discovery.json delete mode 100644 test/data/youtube-playlistid.json delete mode 100644 test/data/youtube-playlistitems.json delete mode 100644 test/lib/.gitignore delete mode 100644 test/lib/README delete mode 100644 test/media/floor/.gitignore delete mode 100644 test/media/photo/.gitignore diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index 8b5d90b1ec..20941359d4 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -17,8 +17,8 @@ } IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" -IDSUBMIT_REPOSITORY_PATH = "test/id/" -IDSUBMIT_STAGING_PATH = "test/staging/" +IDSUBMIT_REPOSITORY_PATH = "/assets/ietfdata/doc/draft/repository" +IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/" AGENDA_PATH = '/assets/www6s/proceedings/' MEETINGHOST_LOGO_PATH = AGENDA_PATH diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index bcd04898ea..5d9859c19b 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -8,7 +8,7 @@ from ietf.settings_postgresqldb import DATABASES # pyflakes:ignore IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" -IDSUBMIT_STAGING_PATH = "test/staging/" +IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/" AGENDA_PATH = '/assets/www6s/proceedings/' MEETINGHOST_LOGO_PATH = AGENDA_PATH @@ -53,7 +53,7 @@ FTP_DIR = '/assets/ftp' NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/' -SLIDE_STAGING_PATH = 'test/staging/' +SLIDE_STAGING_PATH = '/assets/www6s/staging/' DE_GFM_BINARY = '/usr/local/bin/de-gfm' diff --git a/docker/scripts/app-create-dirs.sh b/docker/scripts/app-create-dirs.sh index b75c57767d..50431f4793 100755 --- a/docker/scripts/app-create-dirs.sh +++ b/docker/scripts/app-create-dirs.sh @@ -1,13 +1,6 @@ #!/bin/bash for sub in \ - test/id \ - test/staging \ - test/archive \ - test/rfc \ - test/media \ - test/wiki/ietf \ - data/nomcom_keys/public_keys \ /assets/archive/id \ /assets/collection \ /assets/collection/draft-archive \ @@ -27,6 +20,7 @@ for sub in \ /assets/ietfdata/derived \ /assets/ietfdata/derived/bibxml \ /assets/ietfdata/derived/bibxml/bibxml-ids \ + /assets/ietfdata/doc/draft/repository \ /assets/www6s \ /assets/www6s/staging \ /assets/www6s/wg-descriptions \ diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py index 6199ed7eb5..b15aa70e73 100644 --- a/ietf/meeting/tests_js.py +++ b/ietf/meeting/tests_js.py @@ -5,7 +5,7 @@ import time import datetime import shutil -import os +import tempfile import re from django.utils import timezone @@ -939,13 +939,8 @@ def tearDown(self): def tempdir(self, label): # Borrowed from test_utils.TestCase slug = slugify(self.__class__.__name__.replace('.','-')) - dirname = "tmp-{label}-{slug}-dir".format(**locals()) - if 'VIRTUAL_ENV' in os.environ: - dirname = os.path.join(os.environ['VIRTUAL_ENV'], dirname) - path = os.path.abspath(dirname) - if not os.path.exists(path): - os.mkdir(path) - return path + suffix = "-{label}-{slug}-dir".format(**locals()) + return tempfile.mkdtemp(suffix=suffix) def displayed_interims(self, groups=None): sessions = add_event_info_to_session_qs( diff --git a/ietf/person/factories.py b/ietf/person/factories.py index 2247fa9b2b..2012483c0d 100644 --- a/ietf/person/factories.py +++ b/ietf/person/factories.py @@ -8,7 +8,7 @@ import faker.config import os import random -import shutil +from PIL import Image from unidecode import unidecode from unicodedata import normalize @@ -103,10 +103,9 @@ def default_photo(obj, create, extracted, **kwargs): # pylint: disable=no-self-a media_name = "%s/%s.jpg" % (settings.PHOTOS_DIRNAME, photo_name) obj.photo = media_name obj.photo_thumb = media_name - photosrc = os.path.join(settings.TEST_DATA_DIR, "profile-default.jpg") photodst = os.path.join(settings.PHOTOS_DIR, photo_name + '.jpg') - if not os.path.exists(photodst): - shutil.copy(photosrc, photodst) + img = Image.new('RGB', (200, 200)) + img.save(photodst) def delete_file(file): os.unlink(file) atexit.register(delete_file, photodst) diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 024512a8db..94ca22c71b 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -9,9 +9,12 @@ # ./manage.py test --settings=settings_test doc.ChangeStateTestCase # -import os +import atexit +import os +import shutil +import tempfile from ietf.settings import * # pyflakes:ignore -from ietf.settings import TEST_CODE_COVERAGE_CHECKER, BASE_DIR, PHOTOS_DIRNAME +from ietf.settings import TEST_CODE_COVERAGE_CHECKER import debug # pyflakes:ignore debug.debug = True @@ -48,11 +51,20 @@ def __getitem__(self, item): if TEST_CODE_COVERAGE_CHECKER and not TEST_CODE_COVERAGE_CHECKER._started: # pyflakes:ignore TEST_CODE_COVERAGE_CHECKER.start() # pyflakes:ignore -NOMCOM_PUBLIC_KEYS_DIR=os.path.abspath("tmp-nomcom-public-keys-dir") -MEDIA_ROOT = os.path.join(os.path.dirname(BASE_DIR), 'test/media/') # pyflakes:ignore -MEDIA_URL = '/test/media/' -PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME # pyflakes:ignore +def tempdir_with_cleanup(**kwargs): + """Utility to create a temporary dir and arrange cleanup""" + _dir = tempfile.mkdtemp(**kwargs) + atexit.register(shutil.rmtree, _dir) + return _dir + + +NOMCOM_PUBLIC_KEYS_DIR = tempdir_with_cleanup(suffix="-nomcom-public-keys-dir") + +MEDIA_ROOT = tempdir_with_cleanup(suffix="-media") +PHOTOS_DIRNAME = "photo" +PHOTOS_DIR = os.path.join(MEDIA_ROOT, PHOTOS_DIRNAME) +os.mkdir(PHOTOS_DIR) # Undo any developer-dependent middleware when running the tests MIDDLEWARE = [ c for c in MIDDLEWARE if not c in DEV_MIDDLEWARE ] # pyflakes:ignore diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py index ba35665a8d..86c5a0c1c3 100644 --- a/ietf/utils/test_utils.py +++ b/ietf/utils/test_utils.py @@ -34,7 +34,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os +import tempfile import re import email import html5lib @@ -239,13 +239,8 @@ def normalize(x): def tempdir(self, label): slug = slugify(self.__class__.__name__.replace('.','-')) - dirname = "tmp-{label}-{slug}-dir".format(**locals()) - if 'VIRTUAL_ENV' in os.environ: - dirname = os.path.join(os.environ['VIRTUAL_ENV'], dirname) - path = os.path.abspath(dirname) - if not os.path.exists(path): - os.mkdir(path) - return path + suffix = "-{label}-{slug}-dir".format(**locals()) + return tempfile.mkdtemp(suffix=suffix) def assertNoFormPostErrors(self, response, error_css_selector=".is-invalid"): """Try to fish out form errors, if none found at least check the @@ -306,7 +301,7 @@ def setUp(self): # Replace settings paths with temporary directories. self._ietf_temp_dirs = {} # trashed during tearDown, DO NOT put paths you care about in this - for setting in self.settings_temp_path_overrides: + for setting in set(self.settings_temp_path_overrides): self._ietf_temp_dirs[setting] = self.tempdir(slugify(setting)) self._ietf_saved_context = django.test.utils.override_settings(**self._ietf_temp_dirs) self._ietf_saved_context.enable() diff --git a/media/.gitignore b/media/.gitignore deleted file mode 100644 index dfa8ca37ce..0000000000 --- a/media/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/floor diff --git a/media/photo/nopictureavailable.jpg b/media/photo/nopictureavailable.jpg deleted file mode 100644 index 0895f9f57c74c6199a8eed323d71fe1caae52c9e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1618 zcmeH|+dmTu0LM3)k;R%}$~pEytF;rIK#{T{wAzdTR(6Uqv@3IG5=8S8}c`bFZuk(c|` z37l)*0KhioXnT84ti3(dGco>jbnGbrz`CG-5FDjP&_s16MH=lH!^a-VId>PWyjQu@ z=THr+BV%?J5l6=&rVM#R@#nD{Psgk+J2z`Q96Zx3mMb;;pR{K}jlZf+qgdwG6!uO{ zK#ftX?-Wnom0Er|(y~0dqj~Ps=QLrJ;8yDAN09;$?@_%Uy+jiTuDEBa4d*1~7SWEG z6jO7nwdHDjaS)T>I#(HlhvNAMf%EpgsUdG)&2BLo+Z`cV(6jpCtsn0y(u#mz?hrM7-%sLZuf(pZ);jyrh8$_!!!CwW8+(-yM}C7WZF= zRJ+CS>v@ZRhFAVETiu)>2-{+idZKcKq`o`jrs>S(#*nYoIlT6F;?h&jEzp3S)3_~5cZI+uzN!Z@b1aVzdKe9vS zp5*@a!vSk+$Zg$u9f1|ALx~a+{?;0Iartn?NNNg3y74&0NlZ>;9Qm}A#uHNMrrV`| zUoWF*^;2jWHIBAoE@!KG##v2PCq{P>n_+hKFO@*^_rauE-nfgBw$+5b3z)x=W$U0vN7n?|9 zZokX;;JezMI27=_v&FSIYv{$8`GBa&D6-*v3EJ zOvRCT9kda%?xW*gfMJR#qFffNYaiKobFgca{ER$G9wYYz%akBTs@}5~%o6mILI%Y# zLfwG*RB+19_Q)mr8K$#=QLNT(aAGfF0T%{e+ovQSVTYT{qBrs-V21gvV-S@H{f~Df zL2NT0V4kE|M&!0dAb2k&l15Z>P&`U-0duX!wbMXTYFc#XQ| z30u|N*4pH1l8i(Oc~uwJ;|JFv(c+75;IOA3qaS)%o;L^+d2Mh`y9wxNS$h# z?tzpzr76Cf__iW`vH`!4zB#cp)1SoTnr1$*LRLMQ7o2;II{u>p5Lu7`tY)MUJVUqQ z0RR~Q7Gvvu0r;uZLZR=V=D^fd{(2^A2py^jjCW{&8()!)FE06#D%d1{JvP|QwXj+; zY|r9f@Hd}iV#4az7686bOo)NGF$DB2}c8==|f?7-9&`9RxkOH`QrTlKbV z`4>C<)mn^VLF%_PTA;)elM04T`-rx~h4-~E#qPt|#gH%)8no*Fq5r@UKi@Jb9qAz3 erH0!=i&Q7I<(v2jXU|_>0l+%AWA537X8#0-_TYm6 diff --git a/media/photo/profile-default.jpg b/media/photo/profile-default.jpg deleted file mode 100644 index d6b03e100415a0fa9effad25c793f33957929e2d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1397 zcmex=>ukC3pCfH06P05XITq?4J21E^7eo0A(TN+S4wfI*Oh zA%!7@nNf*>Nsy6Qkn#T!25GQUnSsuQ0}gg37NE<3k~RVijLbkcb24+YLKHAEFfjuq z*##916FG#0B^(2Tr#w_DY!neSG7m~hE-G2LQQ6pe;^vD$-6F__LQDsV3L;qr6}`p4 z!wfW8kXewyp5ekpK}JCV!P>Ier_+?rbWdKHE7*~#p#0QyQ}e|`{OR`!-(>uFVYbSi z_fx^?Y8{@R=lZJ8ddOYb8nkom^=1i&%Z=CXH95KFgg%)Tdd($qGTX%sg3C5qe%w(L z^&rfD#+(`RU43>?GDyA zi-2ZxuAlNP?sD`i&5v(aF-m8r{5D5cMQ!(_%4O9t>yW>v~RuU`$cX| zNhwBKzKfkX{3pv!OW(o8g~bI(ur4nu_;qdjzj@35xu$7*Joy+ZYLL*YKNsA;djUZg*#PcpextM7Qv@%!X?FPsG&04%Owb|~v@3`+>FnaqFt^rGa3S*v`bS?p)t z{lm7riGOx>%y+h}OD2^t@Ge=qSl7}bZX>^I`LVWW-yJWWU1nVBbGXC)w%mV)?1p>W zc($GRqun4aTeC*o`+(G}lQT>@{t26#-`{$tTCA`6?BO4I$6IqZ-YWRJO-d``yOG)3 z!ry+@cPb`%eVHgAD99E+Gm^W_@Vs~NHnu2zy{pG6X8zT@`=dv8`CHz-zupCXGdwV3 zK2uCx-jTBF2V(qPjcZq=F>md^Q+xY(2G_?}>8pJgJJx=>l|Su;cBY!#ouKW3?`0SC z9WrZ?pTAX5!f5^i+ygEl6=8`h6x3~;%6paa#b<1l3ZL;WO#I4 zc3bO`e3mj1nLt6t=$+;wfvIh>f^OH?15J<5GP!zWuK%7Dof+SF{+*FAPg}g-RVtd+rSaJ=Std8L zR)nwIB=cNZd**v>ukC3pCfH06P05XITq?4J21E^7eo0A(TN+S4wfI*Oh zA%!7@nNf*>Nsy6Qkn#T!25GQUnSsuQ0}gg37NE<3k~RVijLbkcb24+YLKHAEFfjuq z*##916FG#0B^(2Tr#w_DY!neSG7m~hE-G2LQQ6pe;^vD$-6F__LQDsV3L;qr6}`p4 z!wfW8kXewyp5ekpK}JCV!P>Ier_+?rbWdKHE7*~#p#0QyQ}e|`{OR`!-(>uFVYbSi z_fx^?Y8{@R=lZJ8ddOYb8nkom^=1i&%Z=CXH95KFgg%)Tdd($qGTX%sg3C5qe%w(L z^&rfD#+(`RU43>?GDyA zi-2ZxuAlNP?sD`i&5v(aF-m8r{5D5cMQ!(_%4O9t>yW>v~RuU`$cX| zNhwBKzKfkX{3pv!OW(o8g~bI(ur4nu_;qdjzj@35xu$7*Joy+ZYLL*YKNsA;djUZg*#PcpextM7Qv@%!X?FPsG&04%Owb|~v@3`+>FnaqFt^rGa3S*v`bS?p)t z{lm7riGOx>%y+h}OD2^t@Ge=qSl7}bZX>^I`LVWW-yJWWU1nVBbGXC)w%mV)?1p>W zc($GRqun4aTeC*o`+(G}lQT>@{t26#-`{$tTCA`6?BO4I$6IqZ-YWRJO-d``yOG)3 z!ry+@cPb`%eVHgAD99E+Gm^W_@Vs~NHnu2zy{pG6X8zT@`=dv8`CHz-zupCXGdwV3 zK2uCx-jTBF2V(qPjcZq=F>md^Q+xY(2G_?}>8pJgJJx=>l|Su;cBY!#ouKW3?`0SC z9WrZ?pTAX5!f5^i+ygEl6=8`h6x3~;%6paa#b<1l3ZL;WO#I4 zc3bO`e3mj1nLt6t=$+;wfvIh>f^OH?15J<5GP!zWuK%7Dof+SF{+*FAPg}g-RVtd+rSaJ=Std8L zR)nwIB=cNZd**v