From e50d34c1fe06df0c3a07070b42fc31c7c5a99565 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 24 Oct 2023 15:27:53 -0700 Subject: [PATCH 01/22] fix: Reorder conflict review columns Fixes #6528 Also remove some redundant computation while I'm here. --- ietf/doc/views_search.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 4e791aea76..f20f6ab4c5 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -518,8 +518,8 @@ def ad_workload(request): for id, (g, uig) in enumerate( [ - ("AD Review Conflict Review", False), ("Needs Shepherd Conflict Review", False), + ("AD Review Conflict Review", False), ("IESG Evaluation Conflict Review", True), ("Approved Conflict Review", True), ("Withdrawn Conflict Review", None), @@ -574,8 +574,6 @@ def ad_workload(request): ad.dashboard = urlreverse( "ietf.doc.views_search.docs_for_ad", kwargs=dict(name=ad.full_name_as_key()) ) - ad.counts = defaultdict(list) - ad.prev = defaultdict(list) ad.doc_now = defaultdict(list) ad.doc_prev = defaultdict(list) @@ -588,14 +586,11 @@ def ad_workload(request): groups[group_type][group] = len(groups[group_type]) group_names[group_type].append(group) - inc = len(groups[group_type]) - len(ad.counts[group_type]) + inc = len(groups[group_type]) - len(ad.doc_now[group_type]) if inc > 0: - ad.counts[group_type].extend([0] * inc) - ad.prev[group_type].extend([0] * inc) ad.doc_now[group_type].extend(set() for _ in range(inc)) ad.doc_prev[group_type].extend(set() for _ in range(inc)) - ad.counts[group_type][groups[group_type][group]] += 1 ad.doc_now[group_type][groups[group_type][group]].add(doc) last_state_event = ( @@ -606,16 +601,13 @@ def ad_workload(request): .first() ) if (last_state_event is not None) and (right_now - last_state_event.time) > delta: - ad.prev[group_type][groups[group_type][group]] += 1 ad.doc_prev[group_type][groups[group_type][group]].add(doc) for ad in ads: ad.doc_diff = defaultdict(list) for gt in group_types: - inc = len(groups[gt]) - len(ad.counts[gt]) + inc = len(groups[gt]) - len(ad.doc_now[gt]) if inc > 0: - ad.counts[gt].extend([0] * inc) - ad.prev[gt].extend([0] * inc) ad.doc_now[gt].extend([set()] * inc) ad.doc_prev[gt].extend([set()] * inc) @@ -642,8 +634,8 @@ def ad_workload(request): [ ( group_names[gt][index], - ad.counts[gt][index], - ad.prev[gt][index], + len(ad.doc_now[gt][index]), + len(ad.doc_prev[gt][index]), ad.doc_diff[gt][index], ) for index in range(len(group_names[gt])) @@ -654,8 +646,8 @@ def ad_workload(request): sums=[ ( group_names[gt][index], - sum([ad.counts[gt][index] for ad in ads]), - sum([ad.prev[gt][index] for ad in ads]), + sum([len(ad.doc_now[gt][index]) for ad in ads]), + sum([len(ad.doc_prev[gt][index]) for ad in ads]), ) for index in range(len(group_names[gt])) ], From a2efb79a8e48727986f3752b8bf347342a6efaf7 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 24 Oct 2023 15:49:40 -0700 Subject: [PATCH 02/22] Remove some more stuff that isn't needed --- ietf/doc/views_search.py | 99 +++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 58 deletions(-) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index f20f6ab4c5..d42e98b086 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -345,44 +345,35 @@ def ad_dashboard_group(doc): if doc.get_state_slug('draft') == 'rfc': return 'RFC' elif doc.get_state_slug('draft') == 'active' and doc.get_state_slug('draft-iesg'): - return '%s Internet-Draft' % doc.get_state('draft-iesg').name + return str(doc.get_state('draft-iesg').name) else: - return '%s Internet-Draft' % doc.get_state('draft').name + return str(doc.get_state('draft').name) elif doc.type.slug=='conflrev': if doc.get_state_slug('conflrev') in ('appr-reqnopub-sent','appr-noprob-sent'): - return 'Approved Conflict Review' + return 'Approved' elif doc.get_state_slug('conflrev') in ('appr-reqnopub-pend','appr-noprob-pend','appr-reqnopub-pr','appr-noprob-pr'): - return "%s Conflict Review" % State.objects.get(type__slug='draft-iesg',slug='approved') + return str(State.objects.get(type__slug='draft-iesg',slug='approved')) else: - return '%s Conflict Review' % doc.get_state('conflrev') + return str(doc.get_state('conflrev')) elif doc.type.slug=='statchg': if doc.get_state_slug('statchg') in ('appr-sent',): - return 'Approved Status Change' + return 'Approved' if doc.get_state_slug('statchg') in ('appr-pend','appr-pr'): - return '%s Status Change' % State.objects.get(type__slug='draft-iesg',slug='approved') + return str(State.objects.get(type__slug='draft-iesg',slug='approved')) else: - return '%s Status Change' % doc.get_state('statchg') + return str(doc.get_state('statchg')) elif doc.type.slug=='charter': if doc.get_state_slug('charter') == 'approved': - return "Approved Charter" + return "Approved" else: - return '%s Charter' % doc.get_state('charter') + return str(doc.get_state('charter')) else: return "Document" def shorten_group_name(name): - for s in [ - " Internet-Draft", - " Conflict Review", - " Status Change", - " (Internal Steering Group/IAB Review) Charter", - "Charter", - ]: - if name.endswith(s): - name = name[: -len(s)] - for pat, sub in [ + (r" \(Internal Steering Group/IAB Review\)", ""), ("Writeup", "Write-up"), ("Requested", "Req"), ("Evaluation", "Eval"), @@ -443,14 +434,6 @@ def ad_dashboard_sort_key(doc): if seed.startswith('Needs Shepherd'): return "100%s" % seed - if seed.endswith(' Document'): - seed = seed[:-9] - elif seed.endswith(' Internet-Draft'): - seed = seed[:-15] - elif seed.endswith(' Conflict Review'): - seed = seed[:-16] - elif seed.endswith(' Status Change'): - seed = seed[:-14] state = State.objects.filter(type__slug='draft-iesg',name=seed) if state: ageseconds = 0 @@ -495,34 +478,34 @@ def ad_workload(request): # FIXME: This should really use the database states instead of replicating the logic for id, (g, uig) in enumerate( [ - ("Publication Requested Internet-Draft", False), - ("AD Evaluation Internet-Draft", False), - ("Last Call Requested Internet-Draft", True), - ("In Last Call Internet-Draft", True), - ("Waiting for Writeup Internet-Draft", False), - ("IESG Evaluation - Defer Internet-Draft", False), - ("IESG Evaluation Internet-Draft", True), - ("Waiting for AD Go-Ahead Internet-Draft", False), - ("Approved-announcement to be sent Internet-Draft", True), - ("Approved-announcement sent Internet-Draft", True), + ("Publication Requested", False), + ("AD Evaluation", False), + ("Last Call Requested", True), + ("In Last Call", True), + ("Waiting for Writeup", False), + ("IESG Evaluation - Defer", False), + ("IESG Evaluation", True), + ("Waiting for AD Go-Ahead", False), + ("Approved-announcement to be sent", True), + ("Approved-announcement sent", True), ] ): groups["I-D"][g] = id group_names["I-D"].append(g) up_is_good[g] = uig - for id, g in enumerate(["RFC Ed Queue Internet-Draft", "RFC"]): + for id, g in enumerate(["RFC Ed Queue", "RFC"]): groups["RFC"][g] = id group_names["RFC"].append(g) up_is_good[g] = True for id, (g, uig) in enumerate( [ - ("Needs Shepherd Conflict Review", False), - ("AD Review Conflict Review", False), - ("IESG Evaluation Conflict Review", True), - ("Approved Conflict Review", True), - ("Withdrawn Conflict Review", None), + ("Needs Shepherd", False), + ("AD Review", False), + ("IESG Evaluation", True), + ("Approved", True), + ("Withdrawn", None), ] ): groups["Conflict Review"][g] = id @@ -531,13 +514,13 @@ def ad_workload(request): for id, (g, uig) in enumerate( [ - ("Publication Requested Status Change", False), - ("AD Evaluation Status Change", False), - ("Last Call Requested Status Change", True), - ("In Last Call Status Change", True), - ("Waiting for Writeup Status Change", False), - ("IESG Evaluation Status Change", True), - ("Waiting for AD Go-Ahead Status Change", False), + ("Publication Requested", False), + ("AD Evaluation", False), + ("Last Call Requested", True), + ("In Last Call", True), + ("Waiting for Writeup", False), + ("IESG Evaluation", True), + ("Waiting for AD Go-Ahead", False), ] ): groups["Status Change"][g] = id @@ -546,13 +529,13 @@ def ad_workload(request): for id, (g, uig) in enumerate( [ - ("Not currently under review Charter", None), - ("Draft Charter Charter", None), - ("Start Chartering/Rechartering (Internal Steering Group/IAB Review) Charter", False), - ("External Review (Message to Community, Selected by Secretariat) Charter", True), - ("IESG Review (Charter for Approval, Selected by Secretariat) Charter", True), - ("Approved Charter", True), - ("Replaced Charter", None), + ("Not currently under review", None), + ("Draft Charter", None), + ("Start Chartering/Rechartering (Internal Steering Group/IAB Review)", False), + ("External Review (Message to Community, Selected by Secretariat)", True), + ("IESG Review (Charter for Approval, Selected by Secretariat)", True), + ("Approved", True), + ("Replaced", None), ] ): groups["Charter"][g] = id From 32c117812e313175cdd3e4dd56859246ceca6a7a Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 26 Oct 2023 18:01:10 -0700 Subject: [PATCH 03/22] Progress --- ietf/doc/views_search.py | 467 +++++++++++++------------------ ietf/templates/doc/ad_count.html | 2 +- ietf/templates/doc/ad_list.html | 18 +- 3 files changed, 198 insertions(+), 289 deletions(-) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index d42e98b086..6a508ae5df 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -36,6 +36,7 @@ import re import datetime +import copy from collections import defaultdict @@ -313,65 +314,29 @@ def cached_redirect(cache_key, url): return cached_redirect(cache_key, urlreverse('ietf.doc.views_search.search') + search_args) -def ad_dashboard_group_type(doc): - # Return group type for document for dashboard. - # If doc is not defined return list of all possible - # group types - if not doc: - return ('I-D', 'RFC', 'Conflict Review', 'Status Change', 'Charter') - if doc.type.slug=='draft': - if doc.get_state_slug('draft') == 'rfc': - return 'RFC' - elif doc.get_state_slug('draft') == 'active' and doc.get_state_slug('draft-iesg') and doc.get_state('draft-iesg').name =='RFC Ed Queue': - return 'RFC' - elif doc.get_state_slug('draft') == 'active' and doc.get_state_slug('draft-iesg') and doc.get_state('draft-iesg').name in ('Dead', 'I-D Exists', 'AD is watching'): - return None - elif doc.get_state('draft').name in ('Expired', 'Replaced'): - return None - else: - return 'I-D' - elif doc.type.slug=='conflrev': - return 'Conflict Review' - elif doc.type.slug=='statchg': - return 'Status Change' - elif doc.type.slug=='charter': - return "Charter" - else: - return "Document" -def ad_dashboard_group(doc): +def doc_state(doc): + dt = doc.type.slug + ds = doc.get_state(dt) + if dt == "draft": + dis = doc.get_state("draft-iesg") + if ds.slug == "active" and dis: + return dis.slug + return ds.slug - if doc.type.slug=='draft': - if doc.get_state_slug('draft') == 'rfc': - return 'RFC' - elif doc.get_state_slug('draft') == 'active' and doc.get_state_slug('draft-iesg'): - return str(doc.get_state('draft-iesg').name) - else: - return str(doc.get_state('draft').name) - elif doc.type.slug=='conflrev': - if doc.get_state_slug('conflrev') in ('appr-reqnopub-sent','appr-noprob-sent'): - return 'Approved' - elif doc.get_state_slug('conflrev') in ('appr-reqnopub-pend','appr-noprob-pend','appr-reqnopub-pr','appr-noprob-pr'): - return str(State.objects.get(type__slug='draft-iesg',slug='approved')) - else: - return str(doc.get_state('conflrev')) - elif doc.type.slug=='statchg': - if doc.get_state_slug('statchg') in ('appr-sent',): - return 'Approved' - if doc.get_state_slug('statchg') in ('appr-pend','appr-pr'): - return str(State.objects.get(type__slug='draft-iesg',slug='approved')) - else: - return str(doc.get_state('statchg')) - elif doc.type.slug=='charter': - if doc.get_state_slug('charter') == 'approved': - return "Approved" - else: - return str(doc.get_state('charter')) + +def state_name(doc_type, state, shorten=True): + name = "" + if doc_type in ["draft", "rfc"] and state != "rfc": + name = State.objects.get(type="draft-iesg", slug=state).name + elif state == "rfc": + name = "RFC" else: - return "Document" + name = State.objects.get(type=doc_type, slug=state).name + if not shorten: + return name -def shorten_group_name(name): for pat, sub in [ (r" \(Internal Steering Group/IAB Review\)", ""), ("Writeup", "Write-up"), @@ -381,69 +346,93 @@ def shorten_group_name(name): ("Waiting", "Wait"), ("Go-Ahead", "OK"), ("Approved-", "App, "), + ("Approved No Problem", "App."), ("announcement", "ann."), ("IESG Eval - ", ""), ("Not currently under review", "Not under review"), ("External Review", "Ext. Review"), - (r"IESG Review \(Charter for Approval, Selected by Secretariat\)", "IESG Review"), + ( + r"IESG Review \(Charter for Approval, Selected by Secretariat\)", + "IESG Review", + ), ("Needs Shepherd", "Needs Shep."), ("Approved", "App."), ("Replaced", "Repl."), ("Withdrawn", "Withd."), ("Chartering/Rechartering", "Charter"), - (r"\(Message to Community, Selected by Secretariat\)", "") + (r"\(Message to Community, Selected by Secretariat\)", ""), ]: name = re.sub(pat, sub, name) return name.strip() -def ad_dashboard_sort_key(doc): - - if doc.type.slug=='draft' and doc.get_state_slug('draft') == 'rfc': - return "21%04d" % int(doc.rfc_number()) - if doc.type.slug=='statchg' and doc.get_state_slug('statchg') == 'appr-sent': - return "22%d" % 0 # TODO - get the date of the transition into this state here - if doc.type.slug=='conflrev' and doc.get_state_slug('conflrev') in ('appr-reqnopub-sent','appr-noprob-sent'): - return "23%d" % 0 # TODO - get the date of the transition into this state here - if doc.type.slug=='charter' and doc.get_state_slug('charter') == 'approved': - return "24%d" % 0 # TODO - get the date of the transition into this state here - - seed = ad_dashboard_group(doc) - - if doc.type.slug=='conflrev' and doc.get_state_slug('conflrev') == 'adrev': - state = State.objects.get(type__slug='draft-iesg',slug='ad-eval') - return "1%d%s" % (state.order,seed) - - if doc.type.slug=='charter' and doc.get_state_slug('charter') != 'replaced': - if doc.get_state_slug('charter') in ('notrev','infrev'): - return "100%s" % seed - elif doc.get_state_slug('charter') == 'intrev': - state = State.objects.get(type__slug='draft-iesg',slug='ad-eval') - return "1%d%s" % (state.order,seed) - elif doc.get_state_slug('charter') == 'extrev': - state = State.objects.get(type__slug='draft-iesg',slug='lc') - return "1%d%s" % (state.order,seed) - elif doc.get_state_slug('charter') == 'iesgrev': - state = State.objects.get(type__slug='draft-iesg',slug='iesg-eva') - return "1%d%s" % (state.order,seed) - - if doc.type.slug=='statchg' and doc.get_state_slug('statchg') == 'adrev': - state = State.objects.get(type__slug='draft-iesg',slug='ad-eval') - return "1%d%s" % (state.order,seed) - - if seed.startswith('Needs Shepherd'): - return "100%s" % seed - state = State.objects.filter(type__slug='draft-iesg',name=seed) - if state: - ageseconds = 0 - changetime= doc.latest_event(type='changed_document') - if changetime: - ad = (timezone.now()-doc.latest_event(type='changed_document').time) - ageseconds = (ad.microseconds + (ad.seconds + ad.days * 24 * 3600) * 10**6) / 10**6 - return "1%d%s%s%010d" % (state[0].order,seed,doc.type.slug,ageseconds) - - return "3%s" % seed +def doc_type_name(doc_type): + if doc_type == "rfc": + return "RFC" + if doc_type == "draft": + return "Internet-Draft" + return DocTypeName.objects.get(slug=doc_type).name + + +STATE_SLUGS = { + "draft": [ + "pub-req", + "ad-eval", + "lc-req", + "lc", + "writeupw", + "defer", + "iesg-eva", + "goaheadw", + "approved", + "ann", + ], + "rfc": ["rfcqueue", "rfc"], + "conflrev": [ + "needshep", + "adrev", + "iesgeval", + # "appr-reqnopub-pend", + # "appr-noprob-pend", + # "appr-reqnopub-sent", + "appr-noprob-sent", + # "appr-reqnopub-pr", + # "appr-noprob-pr", + "withdraw", + ], + "statchg": [ + "needshep", + "adrev", + "lc-req", + "in-lc", + "iesgeval", + "goahead", + # "appr-pr", + # "appr-pend", + "appr-sent", + "dead", + ], + "charter": [ + "notrev", + "infrev", + "intrev", + "extrev", + "iesgrev", + "approved", + "replaced", + ], +} + + +def doc_type(doc): + dt = doc.type.slug + if ( + doc.get_state_slug("draft") == "rfc" + or doc.get_state_slug("draft-iesg") == "rfcqueue" + ): + dt = "rfc" + return dt def ad_workload(request): @@ -463,201 +452,97 @@ def ad_workload(request): if p in get_active_ads(): ads.append(p) - doctypes = list( - DocTypeName.objects.filter(used=True) - .exclude(slug__in=("draft", "liai-att")) - .values_list("pk", flat=True) - ) - - up_is_good = {} - group_types = ad_dashboard_group_type(None) - groups = {g: {} for g in group_types} - group_names = {g: [] for g in group_types} - - # Prefill groups in preferred sort order - # FIXME: This should really use the database states instead of replicating the logic - for id, (g, uig) in enumerate( - [ - ("Publication Requested", False), - ("AD Evaluation", False), - ("Last Call Requested", True), - ("In Last Call", True), - ("Waiting for Writeup", False), - ("IESG Evaluation - Defer", False), - ("IESG Evaluation", True), - ("Waiting for AD Go-Ahead", False), - ("Approved-announcement to be sent", True), - ("Approved-announcement sent", True), - ] - ): - groups["I-D"][g] = id - group_names["I-D"].append(g) - up_is_good[g] = uig - - for id, g in enumerate(["RFC Ed Queue", "RFC"]): - groups["RFC"][g] = id - group_names["RFC"].append(g) - up_is_good[g] = True - - for id, (g, uig) in enumerate( - [ - ("Needs Shepherd", False), - ("AD Review", False), - ("IESG Evaluation", True), - ("Approved", True), - ("Withdrawn", None), - ] - ): - groups["Conflict Review"][g] = id - group_names["Conflict Review"].append(g) - up_is_good[g] = uig - - for id, (g, uig) in enumerate( - [ - ("Publication Requested", False), - ("AD Evaluation", False), - ("Last Call Requested", True), - ("In Last Call", True), - ("Waiting for Writeup", False), - ("IESG Evaluation", True), - ("Waiting for AD Go-Ahead", False), - ] - ): - groups["Status Change"][g] = id - group_names["Status Change"].append(g) - up_is_good[g] = uig - - for id, (g, uig) in enumerate( - [ - ("Not currently under review", None), - ("Draft Charter", None), - ("Start Chartering/Rechartering (Internal Steering Group/IAB Review)", False), - ("External Review (Message to Community, Selected by Secretariat)", True), - ("IESG Review (Charter for Approval, Selected by Secretariat)", True), - ("Approved", True), - ("Replaced", None), - ] - ): - groups["Charter"][g] = id - group_names["Charter"].append(g) - up_is_good[g] = uig - for ad in ads: - form = SearchForm( - { - "by": "ad", - "ad": ad.id, - "rfcs": "on", - "activedrafts": "on", - "olddrafts": "on", - "doctypes": doctypes, - } - ) - ad.dashboard = urlreverse( "ietf.doc.views_search.docs_for_ad", kwargs=dict(name=ad.full_name_as_key()) ) - ad.doc_now = defaultdict(list) - ad.doc_prev = defaultdict(list) - - for doc in retrieve_search_results(form): - group_type = ad_dashboard_group_type(doc) - if group_type and group_type in groups: - # Right now, anything with group_type "Document", such as a bofreq is not handled. - group = ad_dashboard_group(doc) - if group not in groups[group_type]: - groups[group_type][group] = len(groups[group_type]) - group_names[group_type].append(group) - - inc = len(groups[group_type]) - len(ad.doc_now[group_type]) - if inc > 0: - ad.doc_now[group_type].extend(set() for _ in range(inc)) - ad.doc_prev[group_type].extend(set() for _ in range(inc)) - - ad.doc_now[group_type][groups[group_type][group]].add(doc) - - last_state_event = ( - doc.docevent_set.filter( - Q(type="started_iesg_process") | Q(type="changed_state") - ) - .order_by("-time") - .first() + ad.doc_now = { + dt: {state: set() for state in STATE_SLUGS[dt]} for dt in STATE_SLUGS + } + ad.doc_prev = copy.deepcopy(ad.doc_now) + ad.doc_diff = copy.deepcopy(ad.doc_now) + + for doc in Document.objects.filter(ad=ad): + dt = doc_type(doc) + state = doc_state(doc) + + if state in ad.doc_now[dt]: + ad.doc_now[dt][state].add(doc) + + last_state_event = ( + doc.docevent_set.filter( + Q(type="started_iesg_process") | Q(type="changed_state") ) - if (last_state_event is not None) and (right_now - last_state_event.time) > delta: - ad.doc_prev[group_type][groups[group_type][group]].add(doc) - - for ad in ads: - ad.doc_diff = defaultdict(list) - for gt in group_types: - inc = len(groups[gt]) - len(ad.doc_now[gt]) - if inc > 0: - ad.doc_now[gt].extend([set()] * inc) - ad.doc_prev[gt].extend([set()] * inc) - - ad.doc_diff[gt].extend([set()] * len(groups[gt])) - for idx, g in enumerate(group_names[gt]): - ad.doc_diff[gt][idx] = ad.doc_prev[gt][idx] ^ ad.doc_now[gt][idx] - - # Shorten the names of groups - for gt in group_types: - for idx, g in enumerate(group_names[gt]): - group_names[gt][idx] = ( - shorten_group_name(g), - g, - up_is_good[g] if g in up_is_good else None, + .order_by("-time") + .first() ) + if (last_state_event is not None) and ( + right_now - last_state_event.time + ) > delta: + if state in ad.doc_now[dt]: + ad.doc_prev[dt][state].add(doc) + + for dt in STATE_SLUGS: + for state in ad.doc_now[dt]: + ad.doc_diff[dt][state] = ad.doc_prev[dt][state] ^ ad.doc_now[dt][state] workload = [ dict( - group_type=gt, - group_names=group_names[gt], + doc_type=doc_type_name(dt), + state_names=[state_name(dt, state) for state in ad.doc_now[dt]], counts=[ ( ad, [ ( - group_names[gt][index], - len(ad.doc_now[gt][index]), - len(ad.doc_prev[gt][index]), - ad.doc_diff[gt][index], + state, + len(ad.doc_now[dt][state]), + len(ad.doc_prev[dt][state]), + ad.doc_diff[dt][state], ) - for index in range(len(group_names[gt])) + for state in ad.doc_now[dt] ], ) for ad in ads ], sums=[ ( - group_names[gt][index], - sum([len(ad.doc_now[gt][index]) for ad in ads]), - sum([len(ad.doc_prev[gt][index]) for ad in ads]), + state, + sum([len(ad.doc_now[dt][state]) for ad in ads]), + sum([len(ad.doc_prev[dt][state]) for ad in ads]), ) - for index in range(len(group_names[gt])) + for state in ad.doc_now[dt] ], ) - for gt in group_types + for dt in STATE_SLUGS ] return render(request, "doc/ad_list.html", {"workload": workload, "delta": delta}) + def docs_for_ad(request, name): + def sort_key(doc): + key = list(STATE_SLUGS.keys()).index(doc_type(doc)) + return key + ad = None - responsible = Document.objects.values_list('ad', flat=True).distinct() - for p in Person.objects.filter(Q(role__name__in=("pre-ad", "ad"), - role__group__type="area", - role__group__state="active") - | Q(pk__in=responsible)).distinct(): + responsible = Document.objects.values_list("ad", flat=True).distinct() + for p in Person.objects.filter( + Q( + role__name__in=("pre-ad", "ad"), + role__group__type="area", + role__group__state="active", + ) + | Q(pk__in=responsible) + ).distinct(): if name == p.full_name_as_key(): ad = p break if not ad: raise Http404 - form = SearchForm({'by':'ad','ad': ad.id, - 'rfcs':'on', 'activedrafts':'on', 'olddrafts':'on', - 'sort': 'status', - 'doctypes': list(DocTypeName.objects.filter(used=True).exclude(slug__in=('draft','liai-att')).values_list("pk", flat=True))}) - results, meta = prepare_document_table(request, retrieve_search_results(form), form.data, max_results=500) - results.sort(key=ad_dashboard_sort_key) + + results, meta = prepare_document_table(request, Document.objects.filter(ad=ad)) + results.sort(key=lambda d: sort_key(d)) del meta["headers"][-1] # filter out some results @@ -681,28 +566,37 @@ def docs_for_ad(request, name): ] for d in results: - d.search_heading = ad_dashboard_group(d) + dt = d.type.slug + d.search_heading = state_name(dt, doc_state(d), shorten=False) + if d.search_heading != "RFC": + d.search_heading += f" {doc_type_name(dt)}" # Additional content showing docs with blocking positions by this AD, # and docs that the AD hasn't balloted on that are lacking ballot positions to progress blocked_docs = [] not_balloted_docs = [] if ad in get_active_ads(): - iesg_docs = Document.objects.filter(Q(states__type="draft-iesg", - states__slug__in=IESG_BALLOT_ACTIVE_STATES) | - Q(states__type="charter", - states__slug__in=IESG_CHARTER_ACTIVE_STATES) | - Q(states__type__in=("statchg", "conflrev"), - states__slug__in=IESG_STATCHG_CONFLREV_ACTIVE_STATES)).distinct() - possible_docs = iesg_docs.filter(docevent__ballotpositiondocevent__pos__blocking=True, - docevent__ballotpositiondocevent__balloter=ad) + iesg_docs = Document.objects.filter( + Q(states__type="draft-iesg", states__slug__in=IESG_BALLOT_ACTIVE_STATES) + | Q(states__type="charter", states__slug__in=IESG_CHARTER_ACTIVE_STATES) + | Q( + states__type__in=("statchg", "conflrev"), + states__slug__in=IESG_STATCHG_CONFLREV_ACTIVE_STATES, + ) + ).distinct() + possible_docs = iesg_docs.filter( + docevent__ballotpositiondocevent__pos__blocking=True, + docevent__ballotpositiondocevent__balloter=ad, + ) for doc in possible_docs: ballot = doc.active_ballot() if not ballot: continue blocking_positions = [p for p in ballot.all_positions() if p.pos.blocking] - if not blocking_positions or not any( p.balloter==ad for p in blocking_positions ): + if not blocking_positions or not any( + p.balloter == ad for p in blocking_positions + ): continue augment_events_with_revision(doc, blocking_positions) @@ -714,7 +608,12 @@ def docs_for_ad(request, name): # latest first if blocked_docs: - blocked_docs.sort(key=lambda d: min(p.time for p in d.blocking_positions if p.balloter==ad), reverse=True) + blocked_docs.sort( + key=lambda d: min( + p.time for p in d.blocking_positions if p.balloter == ad + ), + reverse=True, + ) possible_docs = iesg_docs.exclude( Q(docevent__ballotpositiondocevent__balloter=ad) @@ -735,9 +634,19 @@ def docs_for_ad(request, name): if re.search(r"\bNeeds\s+\d+", iesg_ballot_summary): not_balloted_docs.append(doc) - return render(request, 'doc/drafts_for_ad.html', { - 'form':form, 'docs':results, 'meta':meta, 'ad_name': ad.plain_name(), 'blocked_docs': blocked_docs, 'not_balloted_docs': not_balloted_docs - }) + return render( + request, + "doc/drafts_for_ad.html", + { + "docs": results, + "meta": meta, + "ad_name": ad.name, + "blocked_docs": blocked_docs, + "not_balloted_docs": not_balloted_docs, + }, + ) + + def drafts_in_last_call(request): lc_state = State.objects.get(type="draft-iesg", slug="lc").pk form = SearchForm({'by':'state','state': lc_state, 'rfcs':'on', 'activedrafts':'on'}) @@ -801,7 +710,7 @@ def recent_drafts(request, days=7): def index_all_drafts(request): # try to be efficient since this view returns a lot of data - categories = [] + states = [] for s in ("active", "rfc", "expired", "repl", "auth-rm", "ietf-rm"): state = State.objects.get(type="draft", slug=s) @@ -835,12 +744,12 @@ def index_all_drafts(request): names = [f'{n}' for n, __ in names if n not in names_to_skip] - categories.append((state, + states.append((state, heading, len(names), "
".join(names) )) - return render(request, 'doc/index_all_drafts.html', { "categories": categories }) + return render(request, 'doc/index_all_drafts.html', { "states": states }) def index_active_drafts(request): slowcache = caches['slowpages'] diff --git a/ietf/templates/doc/ad_count.html b/ietf/templates/doc/ad_count.html index f69b0cc838..08a51c6378 100644 --- a/ietf/templates/doc/ad_count.html +++ b/ietf/templates/doc/ad_count.html @@ -11,7 +11,7 @@ {{ delta.days }} days ago, the count was {{ prev }}. {% if docs_delta %} - {{ group.group_type }}s in the delta are: + {{ type.doc_type }}s in the delta are:
    {% for d in docs_delta %}
  • {{ d.name }}
    • diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html index 189754e8ac..af9fd09799 100644 --- a/ietf/templates/doc/ad_list.html +++ b/ietf/templates/doc/ad_list.html @@ -15,29 +15,29 @@

    Area Director Workload

    are only shown to logged-in Area Directors. {% endif %} - {% for group in workload %} -

    {{ group.group_type }} State Counts

    + {% for type in workload %} +

    {{ type.doc_type }} State Counts

    - {% for g, desc, up_is_good in group.group_names %} - {% endfor %} - {% for ad, ad_data in group.counts %} + {% for ad, ad_data in type.counts %} {% for label, count, prev, docs_delta in ad_data %} {% endfor %} @@ -47,7 +47,7 @@

    {{ group.group_type }} Stat

    - {% for label, count, prev in group.sums %} + {% for label, count, prev in type.sums %} From 0dbfb00e3024165f04f1a06faf6a0ee15697d161 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Fri, 27 Oct 2023 09:35:17 -0700 Subject: [PATCH 04/22] Delivers current functionality --- ietf/doc/views_search.py | 66 ++++++++++++++++---------------- ietf/templates/doc/ad_count.html | 18 ++++----- ietf/templates/doc/ad_list.html | 6 +-- 3 files changed, 45 insertions(+), 45 deletions(-) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 6a508ae5df..a28f5a6d2b 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -377,50 +377,50 @@ def doc_type_name(doc_type): STATE_SLUGS = { "draft": [ - "pub-req", - "ad-eval", - "lc-req", - "lc", - "writeupw", - "defer", - "iesg-eva", - "goaheadw", - "approved", - "ann", + ("pub-req", False), + ("ad-eval", False), + ("lc-req", True), + ("lc", True), + ("writeupw", False), + ("defer", False), + ("iesg-eva", True), + ("goaheadw", False), + ("approved", True), + ("ann", True), ], - "rfc": ["rfcqueue", "rfc"], + "rfc": [("rfcqueue", True), ("rfc", None)], "conflrev": [ - "needshep", - "adrev", - "iesgeval", + ("needshep", False), + ("adrev", False), + ("iesgeval", True), # "appr-reqnopub-pend", # "appr-noprob-pend", # "appr-reqnopub-sent", - "appr-noprob-sent", + ("appr-noprob-sent", True), # "appr-reqnopub-pr", # "appr-noprob-pr", - "withdraw", + ("withdraw", None), ], "statchg": [ - "needshep", - "adrev", - "lc-req", - "in-lc", - "iesgeval", - "goahead", + ("needshep", False), + ("adrev", False), + ("lc-req", True), + ("in-lc", True), + ("iesgeval", True), + ("goahead", False), # "appr-pr", # "appr-pend", - "appr-sent", - "dead", + ("appr-sent", True), + ("dead", None), ], "charter": [ - "notrev", - "infrev", - "intrev", - "extrev", - "iesgrev", - "approved", - "replaced", + ("notrev", None), + ("infrev", None), + ("intrev", False), + ("extrev", True), + ("iesgrev", True), + ("approved", True), + ("replaced", None), ], } @@ -457,7 +457,7 @@ def ad_workload(request): "ietf.doc.views_search.docs_for_ad", kwargs=dict(name=ad.full_name_as_key()) ) ad.doc_now = { - dt: {state: set() for state in STATE_SLUGS[dt]} for dt in STATE_SLUGS + dt: {state: set() for state, _ in STATE_SLUGS[dt]} for dt in STATE_SLUGS } ad.doc_prev = copy.deepcopy(ad.doc_now) ad.doc_diff = copy.deepcopy(ad.doc_now) @@ -496,6 +496,7 @@ def ad_workload(request): [ ( state, + {s: uig for s, uig in STATE_SLUGS[dt]}[state], len(ad.doc_now[dt][state]), len(ad.doc_prev[dt][state]), ad.doc_diff[dt][state], @@ -508,6 +509,7 @@ def ad_workload(request): sums=[ ( state, + {s: uig for s, uig in STATE_SLUGS[dt]}[state], sum([len(ad.doc_now[dt][state]) for ad in ads]), sum([len(ad.doc_prev[dt][state]) for ad in ads]), ) diff --git a/ietf/templates/doc/ad_count.html b/ietf/templates/doc/ad_count.html index 08a51c6378..e40baad34d 100644 --- a/ietf/templates/doc/ad_count.html +++ b/ietf/templates/doc/ad_count.html @@ -19,15 +19,13 @@ {% endif %}" {% endif %} - {% with label.2 as up_is_good %} - {% if prev < count %} - class="bi bi-arrow-up-right-circle{% if count %}-fill{% endif %} {{ up_is_good|yesno:'text-success,text-danger,text-body-secondary' }}" - {% elif prev > count %} - class="bi bi-arrow-down-right-circle{% if count %}-fill{% endif %} {{ up_is_good|yesno:'text-danger,text-success,text-body-secondary' }}" - {% else %} - class="bi bi-arrow-right-circle text-body-secondary" - {% endif %} - > - {% endwith %} + {% if prev < count %} + class="bi bi-arrow-up-right-circle{% if count %}-fill{% endif %} {{ up_is_good|yesno:'text-success,text-danger,text-body-secondary' }}" + {% elif prev > count %} + class="bi bi-arrow-down-right-circle{% if count %}-fill{% endif %} {{ up_is_good|yesno:'text-danger,text-success,text-body-secondary' }}" + {% else %} + class="bi bi-arrow-right-circle text-body-secondary" + {% endif %} + > {% endif %} {% endif %} \ No newline at end of file diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html index af9fd09799..3e944baeb6 100644 --- a/ietf/templates/doc/ad_list.html +++ b/ietf/templates/doc/ad_list.html @@ -35,9 +35,9 @@

    {{ type.doc_type }} State Coun

    - {% for label, count, prev, docs_delta in ad_data %} + {% for label, up_is_good, count, prev, docs_delta in ad_data %} {% endfor %} @@ -47,7 +47,7 @@

    {{ type.doc_type }} State Coun

    - {% for label, count, prev in type.sums %} + {% for label, up_is_good, count, prev in type.sums %} From ac8e7d08ea912cc96534cd24a151f1187f664d76 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Fri, 27 Oct 2023 09:42:35 -0700 Subject: [PATCH 05/22] Add some comments --- ietf/doc/views_search.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index a28f5a6d2b..f81f420915 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -375,6 +375,14 @@ def doc_type_name(doc_type): return DocTypeName.objects.get(slug=doc_type).name +# The document types and state slugs to include in the AD dashboard +# and AD doc list, in the order they should be shown. The Boolean +# indicates whether an upwards trend (compared to a past point in +# time) should be considered a positive development or not. +# +# "rfc" is a custom subset of "draft" that we special-case in the code +# to break out these docs into a separate table. +# STATE_SLUGS = { "draft": [ ("pub-req", False), @@ -388,17 +396,15 @@ def doc_type_name(doc_type): ("approved", True), ("ann", True), ], - "rfc": [("rfcqueue", True), ("rfc", None)], + "rfc": [ + ("rfcqueue", True), + ("rfc", None) + ], "conflrev": [ ("needshep", False), ("adrev", False), ("iesgeval", True), - # "appr-reqnopub-pend", - # "appr-noprob-pend", - # "appr-reqnopub-sent", ("appr-noprob-sent", True), - # "appr-reqnopub-pr", - # "appr-noprob-pr", ("withdraw", None), ], "statchg": [ @@ -408,8 +414,6 @@ def doc_type_name(doc_type): ("in-lc", True), ("iesgeval", True), ("goahead", False), - # "appr-pr", - # "appr-pend", ("appr-sent", True), ("dead", None), ], From 32ef13c1a4be4030264b85d2835e4872f0de3221 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Fri, 27 Oct 2023 09:54:20 -0700 Subject: [PATCH 06/22] Handle expired docs --- ietf/doc/views_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index f81f420915..26edaf57ac 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -327,7 +327,7 @@ def doc_state(doc): def state_name(doc_type, state, shorten=True): name = "" - if doc_type in ["draft", "rfc"] and state != "rfc": + if doc_type in ["draft", "rfc"] and state not in ["rfc", "expired"]: name = State.objects.get(type="draft-iesg", slug=state).name elif state == "rfc": name = "RFC" From deab42331449086e39d3ffa0703c8d0f97d2c4b2 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Fri, 27 Oct 2023 17:23:40 -0700 Subject: [PATCH 07/22] Interim commit --- ietf/doc/tests.py | 14 ++++++++----- ietf/doc/utils_search.py | 30 ++++++++++++++++++++++++++- ietf/doc/views_search.py | 44 ++++++++++++++-------------------------- 3 files changed, 53 insertions(+), 35 deletions(-) diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 106ac17821..c3aca8f7b1 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -36,7 +36,7 @@ from ietf.doc.models import ( Document, DocAlias, DocRelationshipName, RelatedDocument, State, DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent, BallotType, EditedAuthorsDocEvent ) -from ietf.doc.factories import ( DocumentFactory, DocEventFactory, CharterFactory, +from ietf.doc.factories import ( DocumentFactory, DocEventFactory, CharterFactory, ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, IndividualRfcFactory, StateDocEventFactory, BallotPositionDocEventFactory, BallotDocEventFactory, DocumentAuthorFactory, NewRevisionDocEventFactory, @@ -44,7 +44,6 @@ from ietf.doc.forms import NotifyForm from ietf.doc.fields import SearchableDocumentsField from ietf.doc.utils import create_ballot_if_not_open, uppercase_std_abbreviated_name -from ietf.doc.views_search import ad_dashboard_group, ad_dashboard_group_type, shorten_group_name # TODO: red flag that we're importing from views in tests. Move these to utils. from ietf.group.models import Group, Role from ietf.group.factories import GroupFactory, RoleFactory from ietf.ipr.factories import HolderIprDisclosureFactory @@ -60,6 +59,8 @@ from ietf.utils.test_utils import TestCase from ietf.utils.text import normalize_text from ietf.utils.timezone import date_today, datetime_today, DEADLINE_TZINFO, RPC_TZINFO +from ietf.doc.utils_search import doc_type, doc_state +from ietf.doc.utils_search import doc_type_name as get_doc_type_name class SearchTests(TestCase): @@ -297,7 +298,6 @@ def test_ad_workload(self): elif doc_type_name == 'charter': doc = CharterFactory(ad=ad, states=[(doc_type_name, state)]) elif doc_type_name == 'bofreq': - # Note that the view currently doesn't handle bofreqs doc = BofreqFactory(states=[(doc_type_name, state)], bofreqresponsibledocevent__responsible=[ad]) elif doc_type_name == 'conflrev': doc = ConflictReviewFactory(ad=ad, states=State.objects.filter(type_id=doc_type_name, slug=state)) @@ -307,14 +307,18 @@ def test_ad_workload(self): # Currently unreachable doc = DocumentFactory(type_id=doc_type_name, ad=ad, states=[(doc_type_name, state)]) - if not slugify(ad_dashboard_group_type(doc)) in ('document', 'none'): - expected[(slugify(ad_dashboard_group_type(doc)), slugify(ad.full_name_as_key()), slugify(shorten_group_name(ad_dashboard_group(doc))))] += 1 + if not doc_type(doc) in ('document', 'none'): + expected[(slugify(get_doc_type_name(doc_type(doc))), slugify(ad.full_name_as_key()), doc_state(doc))] += 1 url = urlreverse('ietf.doc.views_search.ad_workload') r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) + print(r.content) + print(expected) for group_type, ad, group in expected: + print(group_type, ad, group) + print(q(f'#{group_type}-{ad}-{group}').text()) self.assertEqual(int(q(f'#{group_type}-{ad}-{group}').text()),expected[(group_type, ad, group)]) def test_docs_for_ad(self): diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py index 31aedda0d7..f6d309b6fe 100644 --- a/ietf/doc/utils_search.py +++ b/ietf/doc/utils_search.py @@ -9,7 +9,7 @@ from django.conf import settings -from ietf.doc.models import Document, DocAlias, RelatedDocument, DocEvent, TelechatDocEvent, BallotDocEvent +from ietf.doc.models import Document, DocAlias, RelatedDocument, DocEvent, TelechatDocEvent, BallotDocEvent, DocTypeName from ietf.doc.expire import expirable_drafts from ietf.doc.utils import augment_docs_and_user_with_user_info from ietf.meeting.models import SessionPresentation, Meeting, Session @@ -283,3 +283,31 @@ def num(i): h["sort_url"] = "?" + d.urlencode() return (docs, meta) + + +def doc_type(doc): + dt = doc.type.slug + if ( + doc.get_state_slug("draft") == "rfc" + or doc.get_state_slug("draft-iesg") == "rfcqueue" + ): + dt = "rfc" + return dt + + +def doc_state(doc): + dt = doc.type.slug + ds = doc.get_state(dt) + if dt == "draft": + dis = doc.get_state("draft-iesg") + if ds.slug == "active" and dis: + return dis.slug + return ds.slug + + +def doc_type_name(doc_type): + if doc_type == "rfc": + return "RFC" + if doc_type == "draft": + return "Internet-Draft" + return DocTypeName.objects.get(slug=doc_type).name diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 26edaf57ac..2b87806d4e 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -67,7 +67,7 @@ from ietf.person.utils import get_active_ads from ietf.utils.draft_search import normalize_draftname from ietf.utils.log import log -from ietf.doc.utils_search import prepare_document_table +from ietf.doc.utils_search import prepare_document_table, doc_type, doc_state, doc_type_name class SearchForm(forms.Form): @@ -315,16 +315,6 @@ def cached_redirect(cache_key, url): return cached_redirect(cache_key, urlreverse('ietf.doc.views_search.search') + search_args) -def doc_state(doc): - dt = doc.type.slug - ds = doc.get_state(dt) - if dt == "draft": - dis = doc.get_state("draft-iesg") - if ds.slug == "active" and dis: - return dis.slug - return ds.slug - - def state_name(doc_type, state, shorten=True): name = "" if doc_type in ["draft", "rfc"] and state not in ["rfc", "expired"]: @@ -367,14 +357,6 @@ def state_name(doc_type, state, shorten=True): return name.strip() -def doc_type_name(doc_type): - if doc_type == "rfc": - return "RFC" - if doc_type == "draft": - return "Internet-Draft" - return DocTypeName.objects.get(slug=doc_type).name - - # The document types and state slugs to include in the AD dashboard # and AD doc list, in the order they should be shown. The Boolean # indicates whether an upwards trend (compared to a past point in @@ -384,9 +366,16 @@ def doc_type_name(doc_type): # to break out these docs into a separate table. # STATE_SLUGS = { + "bofreq": [ + ("proposed", None), + ("declined", None), + ("approved", None), + ], "draft": [ + ("idexists", None), ("pub-req", False), ("ad-eval", False), + ("review-e", True), ("lc-req", True), ("lc", True), ("writeupw", False), @@ -403,9 +392,16 @@ def doc_type_name(doc_type): "conflrev": [ ("needshep", False), ("adrev", False), + ("defer", False), ("iesgeval", True), ("appr-noprob-sent", True), + ("appr-reqnopub-pr", True), + ("appr-reqnopub-pend", True), + ("appr-reqnopub-sent", True), + ("appr-noprob-pr", True), + ("appr-noprob-pend", True), ("withdraw", None), + ("dead", None), ], "statchg": [ ("needshep", False), @@ -429,16 +425,6 @@ def doc_type_name(doc_type): } -def doc_type(doc): - dt = doc.type.slug - if ( - doc.get_state_slug("draft") == "rfc" - or doc.get_state_slug("draft-iesg") == "rfcqueue" - ): - dt = "rfc" - return dt - - def ad_workload(request): delta = datetime.timedelta(days=120) right_now = timezone.now() From d55403a97e28e53d684492e3cf985d4cbd00603c Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Mon, 30 Oct 2023 18:57:56 +0200 Subject: [PATCH 08/22] Fix tests --- ietf/doc/tests.py | 91 ++++++++++++++++++++++++++-------------- ietf/doc/utils_search.py | 59 ++++++++++++++++++++++++++ ietf/doc/views_search.py | 88 +++++--------------------------------- 3 files changed, 128 insertions(+), 110 deletions(-) diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index c3aca8f7b1..b93327a901 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -40,7 +40,7 @@ ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, IndividualRfcFactory, StateDocEventFactory, BallotPositionDocEventFactory, BallotDocEventFactory, DocumentAuthorFactory, NewRevisionDocEventFactory, - StatusChangeFactory, BofreqFactory, DocExtResourceFactory, RgDraftFactory) + StatusChangeFactory, DocExtResourceFactory, RgDraftFactory) from ietf.doc.forms import NotifyForm from ietf.doc.fields import SearchableDocumentsField from ietf.doc.utils import create_ballot_if_not_open, uppercase_std_abbreviated_name @@ -59,7 +59,7 @@ from ietf.utils.test_utils import TestCase from ietf.utils.text import normalize_text from ietf.utils.timezone import date_today, datetime_today, DEADLINE_TZINFO, RPC_TZINFO -from ietf.doc.utils_search import doc_type, doc_state +from ietf.doc.utils_search import doc_type, doc_state, AD_WORKLOAD_STATE_SLUGS from ietf.doc.utils_search import doc_type_name as get_doc_type_name @@ -280,42 +280,68 @@ def test_frontpage(self): self.assertContains(r, "Document Search") def test_ad_workload(self): - Role.objects.filter(name_id='ad').delete() - ad = RoleFactory(name_id='ad',group__type_id='area',group__state_id='active',person__name='Example Areadirector').person - doc_type_names = ['bofreq', 'charter', 'conflrev', 'draft', 'statchg'] - expected = defaultdict(lambda :0) - for doc_type_name in doc_type_names: - if doc_type_name=='draft': - states = State.objects.filter(type='draft-iesg', used=True).values_list('slug', flat=True) - else: - states = State.objects.filter(type=doc_type_name, used=True).values_list('slug', flat=True) - - for state in states: - target_num = random.randint(0,2) + Role.objects.filter(name_id="ad").delete() + ad = RoleFactory( + name_id="ad", + group__type_id="area", + group__state_id="active", + person__name="Example Areadirector", + ).person + expected = defaultdict(lambda: 0) + for doc_type_slug in AD_WORKLOAD_STATE_SLUGS: + for state, _ in AD_WORKLOAD_STATE_SLUGS[doc_type_slug]: + target_num = random.randint(0, 2) for _ in range(target_num): - if doc_type_name == 'draft': - doc = IndividualDraftFactory(ad=ad,states=[('draft-iesg', state),('draft','rfc' if state=='pub' else 'active')]) - elif doc_type_name == 'charter': - doc = CharterFactory(ad=ad, states=[(doc_type_name, state)]) - elif doc_type_name == 'bofreq': - doc = BofreqFactory(states=[(doc_type_name, state)], bofreqresponsibledocevent__responsible=[ad]) - elif doc_type_name == 'conflrev': - doc = ConflictReviewFactory(ad=ad, states=State.objects.filter(type_id=doc_type_name, slug=state)) - elif doc_type_name == 'statchg': - doc = StatusChangeFactory(ad=ad, states=State.objects.filter(type_id=doc_type_name, slug=state)) - else: - # Currently unreachable - doc = DocumentFactory(type_id=doc_type_name, ad=ad, states=[(doc_type_name, state)]) - - if not doc_type(doc) in ('document', 'none'): - expected[(slugify(get_doc_type_name(doc_type(doc))), slugify(ad.full_name_as_key()), doc_state(doc))] += 1 + if ( + doc_type_slug == "draft" + or doc_type_slug == "rfc" + and state == "rfcqueue" + ): + doc = IndividualDraftFactory( + ad=ad, + states=[ + ("draft-iesg", state), + ("draft", "rfc" if state == "pub" else "active"), + ], + ) + elif doc_type_slug == "rfc": + doc = WgRfcFactory.create( + states=[("draft", "rfc"), ("draft-iesg", "pub")] + ) + + elif doc_type_slug == "charter": + doc = CharterFactory(ad=ad, states=[(doc_type_slug, state)]) + elif doc_type_slug == "conflrev": + doc = ConflictReviewFactory( + ad=ad, + states=State.objects.filter( + type_id=doc_type_slug, slug=state + ), + ) + elif doc_type_slug == "statchg": + doc = StatusChangeFactory( + ad=ad, + states=State.objects.filter( + type_id=doc_type_slug, slug=state + ), + ) + + url = urlreverse("ietf.doc.views_search.ad_workload") + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + for group_type, ad, group in expected: + print(group_type, ad, group) + print(q(f"#{group_type}-{ad}-{group}").text()) + self.assertEqual( + int(q(f"#{group_type}-{ad}-{group}").text()), + expected[(group_type, ad, group)], + ) url = urlreverse('ietf.doc.views_search.ad_workload') r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - print(r.content) - print(expected) for group_type, ad, group in expected: print(group_type, ad, group) print(q(f'#{group_type}-{ad}-{group}').text()) @@ -398,6 +424,7 @@ def test_indexes(self): rfc = WgRfcFactory() r = self.client.get(urlreverse('ietf.doc.views_search.index_all_drafts')) + print(r.content) self.assertEqual(r.status_code, 200) self.assertContains(r, draft.name) self.assertContains(r, rfc.canonical_name().upper()) diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py index f6d309b6fe..67fa231499 100644 --- a/ietf/doc/utils_search.py +++ b/ietf/doc/utils_search.py @@ -285,6 +285,65 @@ def num(i): return (docs, meta) +# The document types and state slugs to include in the AD dashboard +# and AD doc list, in the order they should be shown. The Boolean +# indicates whether an upwards trend (compared to a past point in +# time) should be considered a positive development or not. +# +# "rfc" is a custom subset of "draft" that we special-case in the code +# to break out these docs into a separate table. +# +AD_WORKLOAD_STATE_SLUGS = { + "draft": [ + ("pub-req", False), + ("ad-eval", False), + ("lc-req", True), + ("lc", True), + ("writeupw", False), + ("defer", False), + ("iesg-eva", True), + ("goaheadw", False), + ("approved", True), + ("ann", True), + ], + "rfc": [ + ("rfcqueue", True), + ("rfc", None) + ], + "conflrev": [ + ("needshep", False), + ("adrev", False), + ("iesgeval", True), + ("appr-noprob-sent", True), + ("appr-reqnopub-pr", True), + ("appr-reqnopub-pend", True), + ("appr-reqnopub-sent", True), + ("appr-noprob-pr", True), + ("appr-noprob-pend", True), + ("withdraw", None), + ], + "statchg": [ + ("needshep", False), + ("adrev", False), + ("lc-req", True), + ("in-lc", True), + ("iesgeval", True), + ("goahead", False), + ("appr-sent", True), + ("dead", None), + ], + "charter": [ + ("notrev", None), + ("infrev", None), + ("intrev", False), + ("extrev", True), + ("iesgrev", True), + ("approved", True), + ("replaced", None), + ], +} + + def doc_type(doc): dt = doc.type.slug if ( diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 2b87806d4e..be1d096e11 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -67,7 +67,7 @@ from ietf.person.utils import get_active_ads from ietf.utils.draft_search import normalize_draftname from ietf.utils.log import log -from ietf.doc.utils_search import prepare_document_table, doc_type, doc_state, doc_type_name +from ietf.doc.utils_search import prepare_document_table, doc_type, doc_state, doc_type_name, AD_WORKLOAD_STATE_SLUGS class SearchForm(forms.Form): @@ -357,74 +357,6 @@ def state_name(doc_type, state, shorten=True): return name.strip() -# The document types and state slugs to include in the AD dashboard -# and AD doc list, in the order they should be shown. The Boolean -# indicates whether an upwards trend (compared to a past point in -# time) should be considered a positive development or not. -# -# "rfc" is a custom subset of "draft" that we special-case in the code -# to break out these docs into a separate table. -# -STATE_SLUGS = { - "bofreq": [ - ("proposed", None), - ("declined", None), - ("approved", None), - ], - "draft": [ - ("idexists", None), - ("pub-req", False), - ("ad-eval", False), - ("review-e", True), - ("lc-req", True), - ("lc", True), - ("writeupw", False), - ("defer", False), - ("iesg-eva", True), - ("goaheadw", False), - ("approved", True), - ("ann", True), - ], - "rfc": [ - ("rfcqueue", True), - ("rfc", None) - ], - "conflrev": [ - ("needshep", False), - ("adrev", False), - ("defer", False), - ("iesgeval", True), - ("appr-noprob-sent", True), - ("appr-reqnopub-pr", True), - ("appr-reqnopub-pend", True), - ("appr-reqnopub-sent", True), - ("appr-noprob-pr", True), - ("appr-noprob-pend", True), - ("withdraw", None), - ("dead", None), - ], - "statchg": [ - ("needshep", False), - ("adrev", False), - ("lc-req", True), - ("in-lc", True), - ("iesgeval", True), - ("goahead", False), - ("appr-sent", True), - ("dead", None), - ], - "charter": [ - ("notrev", None), - ("infrev", None), - ("intrev", False), - ("extrev", True), - ("iesgrev", True), - ("approved", True), - ("replaced", None), - ], -} - - def ad_workload(request): delta = datetime.timedelta(days=120) right_now = timezone.now() @@ -447,7 +379,7 @@ def ad_workload(request): "ietf.doc.views_search.docs_for_ad", kwargs=dict(name=ad.full_name_as_key()) ) ad.doc_now = { - dt: {state: set() for state, _ in STATE_SLUGS[dt]} for dt in STATE_SLUGS + dt: {state: set() for state, _ in AD_WORKLOAD_STATE_SLUGS[dt]} for dt in AD_WORKLOAD_STATE_SLUGS } ad.doc_prev = copy.deepcopy(ad.doc_now) ad.doc_diff = copy.deepcopy(ad.doc_now) @@ -472,7 +404,7 @@ def ad_workload(request): if state in ad.doc_now[dt]: ad.doc_prev[dt][state].add(doc) - for dt in STATE_SLUGS: + for dt in AD_WORKLOAD_STATE_SLUGS: for state in ad.doc_now[dt]: ad.doc_diff[dt][state] = ad.doc_prev[dt][state] ^ ad.doc_now[dt][state] @@ -486,7 +418,7 @@ def ad_workload(request): [ ( state, - {s: uig for s, uig in STATE_SLUGS[dt]}[state], + {s: uig for s, uig in AD_WORKLOAD_STATE_SLUGS[dt]}[state], len(ad.doc_now[dt][state]), len(ad.doc_prev[dt][state]), ad.doc_diff[dt][state], @@ -499,14 +431,14 @@ def ad_workload(request): sums=[ ( state, - {s: uig for s, uig in STATE_SLUGS[dt]}[state], + {s: uig for s, uig in AD_WORKLOAD_STATE_SLUGS[dt]}[state], sum([len(ad.doc_now[dt][state]) for ad in ads]), sum([len(ad.doc_prev[dt][state]) for ad in ads]), ) for state in ad.doc_now[dt] ], ) - for dt in STATE_SLUGS + for dt in AD_WORKLOAD_STATE_SLUGS ] return render(request, "doc/ad_list.html", {"workload": workload, "delta": delta}) @@ -514,7 +446,7 @@ def ad_workload(request): def docs_for_ad(request, name): def sort_key(doc): - key = list(STATE_SLUGS.keys()).index(doc_type(doc)) + key = list(AD_WORKLOAD_STATE_SLUGS.keys()).index(doc_type(doc)) return key ad = None @@ -702,7 +634,7 @@ def recent_drafts(request, days=7): def index_all_drafts(request): # try to be efficient since this view returns a lot of data - states = [] + categories = [] for s in ("active", "rfc", "expired", "repl", "auth-rm", "ietf-rm"): state = State.objects.get(type="draft", slug=s) @@ -736,12 +668,12 @@ def index_all_drafts(request): names = [f'{n}' for n, __ in names if n not in names_to_skip] - states.append((state, + categories.append((state, heading, len(names), "
    ".join(names) )) - return render(request, 'doc/index_all_drafts.html', { "states": states }) + return render(request, 'doc/index_all_drafts.html', { "categories": categories }) def index_active_drafts(request): slowcache = caches['slowpages'] From d8c74a0dbf22d6e30adbc1891386c345867bcd5f Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Mon, 30 Oct 2023 19:01:39 +0200 Subject: [PATCH 09/22] Cleanup --- ietf/doc/tests.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index b93327a901..0ccb7492dc 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -60,7 +60,6 @@ from ietf.utils.text import normalize_text from ietf.utils.timezone import date_today, datetime_today, DEADLINE_TZINFO, RPC_TZINFO from ietf.doc.utils_search import doc_type, doc_state, AD_WORKLOAD_STATE_SLUGS -from ietf.doc.utils_search import doc_type_name as get_doc_type_name class SearchTests(TestCase): @@ -331,8 +330,6 @@ def test_ad_workload(self): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) for group_type, ad, group in expected: - print(group_type, ad, group) - print(q(f"#{group_type}-{ad}-{group}").text()) self.assertEqual( int(q(f"#{group_type}-{ad}-{group}").text()), expected[(group_type, ad, group)], @@ -343,8 +340,6 @@ def test_ad_workload(self): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) for group_type, ad, group in expected: - print(group_type, ad, group) - print(q(f'#{group_type}-{ad}-{group}').text()) self.assertEqual(int(q(f'#{group_type}-{ad}-{group}').text()),expected[(group_type, ad, group)]) def test_docs_for_ad(self): @@ -424,7 +419,6 @@ def test_indexes(self): rfc = WgRfcFactory() r = self.client.get(urlreverse('ietf.doc.views_search.index_all_drafts')) - print(r.content) self.assertEqual(r.status_code, 200) self.assertContains(r, draft.name) self.assertContains(r, rfc.canonical_name().upper()) From b23fae33fd28096a07f3f7e2522b3774a09cccc6 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Mon, 30 Oct 2023 19:06:45 +0200 Subject: [PATCH 10/22] More cleanup --- ietf/doc/tests.py | 12 ++++++------ ietf/doc/views_search.py | 2 -- ietf/templates/doc/ad_list.html | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 0ccb7492dc..2ded666af1 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -59,7 +59,7 @@ from ietf.utils.test_utils import TestCase from ietf.utils.text import normalize_text from ietf.utils.timezone import date_today, datetime_today, DEADLINE_TZINFO, RPC_TZINFO -from ietf.doc.utils_search import doc_type, doc_state, AD_WORKLOAD_STATE_SLUGS +from ietf.doc.utils_search import AD_WORKLOAD_STATE_SLUGS class SearchTests(TestCase): @@ -296,7 +296,7 @@ def test_ad_workload(self): or doc_type_slug == "rfc" and state == "rfcqueue" ): - doc = IndividualDraftFactory( + IndividualDraftFactory( ad=ad, states=[ ("draft-iesg", state), @@ -304,21 +304,21 @@ def test_ad_workload(self): ], ) elif doc_type_slug == "rfc": - doc = WgRfcFactory.create( + WgRfcFactory.create( states=[("draft", "rfc"), ("draft-iesg", "pub")] ) elif doc_type_slug == "charter": - doc = CharterFactory(ad=ad, states=[(doc_type_slug, state)]) + CharterFactory(ad=ad, states=[(doc_type_slug, state)]) elif doc_type_slug == "conflrev": - doc = ConflictReviewFactory( + ConflictReviewFactory( ad=ad, states=State.objects.filter( type_id=doc_type_slug, slug=state ), ) elif doc_type_slug == "statchg": - doc = StatusChangeFactory( + StatusChangeFactory( ad=ad, states=State.objects.filter( type_id=doc_type_slug, slug=state diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index be1d096e11..3709ce9d70 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -38,8 +38,6 @@ import datetime import copy -from collections import defaultdict - from django import forms from django.conf import settings from django.core.cache import cache, caches diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html index 3e944baeb6..717391a0ce 100644 --- a/ietf/templates/doc/ad_list.html +++ b/ietf/templates/doc/ad_list.html @@ -5,10 +5,10 @@ {% block pagehead %} {% endblock %} -{% block title %}Area directors{% endblock %} +{% block title %}IESG Dashboard{% endblock %} {% block content %} {% origin %} -

    Area Director Workload

    +

    IESG Dashboard

    {% if user|has_role:"Area Director,Secretariat" %}
    {{ delta.days }}-day trend indicators From 62aaef723318596c6b14bbb4da9b4f48c768333e Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Mon, 30 Oct 2023 19:18:27 +0200 Subject: [PATCH 11/22] Reduce differences to current view --- ietf/doc/utils_search.py | 12 +++++------- ietf/doc/views_search.py | 2 ++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py index 67fa231499..2e32e8a35b 100644 --- a/ietf/doc/utils_search.py +++ b/ietf/doc/utils_search.py @@ -308,18 +308,13 @@ def num(i): ], "rfc": [ ("rfcqueue", True), - ("rfc", None) + ("rfc", True) ], "conflrev": [ ("needshep", False), ("adrev", False), ("iesgeval", True), - ("appr-noprob-sent", True), - ("appr-reqnopub-pr", True), - ("appr-reqnopub-pend", True), - ("appr-reqnopub-sent", True), - ("appr-noprob-pr", True), - ("appr-noprob-pend", True), + ("approved", True), # synthesized state for all the "appr-" states ("withdraw", None), ], "statchg": [ @@ -361,6 +356,9 @@ def doc_state(doc): dis = doc.get_state("draft-iesg") if ds.slug == "active" and dis: return dis.slug + elif dt == "conflrev": + if ds.slug.startswith("appr"): + return "approved" return ds.slug diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 3709ce9d70..0bc2dccc39 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -319,6 +319,8 @@ def state_name(doc_type, state, shorten=True): name = State.objects.get(type="draft-iesg", slug=state).name elif state == "rfc": name = "RFC" + elif doc_type == "conflrev" and state.startswith("appr"): + name = "Approved" else: name = State.objects.get(type=doc_type, slug=state).name From d0fcbdd7c6a33b374944a23d54d4830afbb40f27 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 2 Nov 2023 15:53:46 +0200 Subject: [PATCH 12/22] Interim commit --- ietf/doc/utils_search.py | 2 +- ietf/doc/views_search.py | 120 ++++++++++++++++++++++++---- ietf/static/css/highcharts.scss | 5 ++ ietf/static/js/highcharts.js | 4 +- ietf/static/js/highstock.js | 2 + ietf/templates/doc/ad_count.html | 5 +- ietf/templates/doc/ad_list.html | 130 +++++++++++++++++++++++++++++-- 7 files changed, 243 insertions(+), 25 deletions(-) diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py index 2e32e8a35b..bcf1968eb0 100644 --- a/ietf/doc/utils_search.py +++ b/ietf/doc/utils_search.py @@ -300,7 +300,7 @@ def num(i): ("lc-req", True), ("lc", True), ("writeupw", False), - ("defer", False), + # ("defer", False), # probably not a useful state to show, since it's rare ("iesg-eva", True), ("goaheadw", False), ("approved", True), diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 0bc2dccc39..8aed77c62e 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -37,6 +37,7 @@ import re import datetime import copy +import dateutil.rrule from django import forms from django.conf import settings @@ -46,8 +47,9 @@ from django.http import Http404, HttpResponseBadRequest, HttpResponse, HttpResponseRedirect, QueryDict from django.shortcuts import render from django.utils import timezone +from django.utils.html import strip_tags from django.utils.cache import _generate_cache_key # type: ignore - +from django.utils.text import slugify import debug # pyflakes:ignore @@ -358,9 +360,24 @@ def state_name(doc_type, state, shorten=True): def ad_workload(request): - delta = datetime.timedelta(days=120) + weeks = 17 + delta = datetime.timedelta(weeks=weeks) right_now = timezone.now() + state_slugs = {} + for dt in AD_WORKLOAD_STATE_SLUGS: + state_slugs[dt] = {} + for ds, _ in AD_WORKLOAD_STATE_SLUGS[dt]: + if dt == "rfc" and ds == "rfc": + state_slugs[dt]["RFC"] = "rfc" + elif dt == "draft" or ds == "rfcqueue": + s = State.objects.get(slug=ds, type="draft-iesg") + elif dt == "conflrev" and ds == "approved": + state_slugs[dt]["Approved"] = ds + else: + s = State.objects.get(slug=ds, type=dt) + state_slugs[dt][s.name] = ds + ads = [] responsible = Document.objects.values_list("ad", flat=True).distinct() for p in Person.objects.filter( @@ -372,17 +389,30 @@ def ad_workload(request): | Q(pk__in=responsible) ).distinct(): if p in get_active_ads(): - ads.append(p) + if p.name == "John Scudder": + ads.append(p) + + dates = list( + dateutil.rrule.rrule( + freq=dateutil.rrule.WEEKLY, count=13, dtstart=right_now - delta + ) + ) + dates.reverse() for ad in ads: ad.dashboard = urlreverse( "ietf.doc.views_search.docs_for_ad", kwargs=dict(name=ad.full_name_as_key()) ) ad.doc_now = { - dt: {state: set() for state, _ in AD_WORKLOAD_STATE_SLUGS[dt]} for dt in AD_WORKLOAD_STATE_SLUGS + dt: {state: set() for state, _ in AD_WORKLOAD_STATE_SLUGS[dt]} + for dt in AD_WORKLOAD_STATE_SLUGS } ad.doc_prev = copy.deepcopy(ad.doc_now) ad.doc_diff = copy.deepcopy(ad.doc_now) + ad.buckets = { + dt: {state: [[] for _ in dates] for state, _ in AD_WORKLOAD_STATE_SLUGS[dt]} + for dt in AD_WORKLOAD_STATE_SLUGS + } for doc in Document.objects.filter(ad=ad): dt = doc_type(doc) @@ -391,26 +421,79 @@ def ad_workload(request): if state in ad.doc_now[dt]: ad.doc_now[dt][state].add(doc) - last_state_event = ( - doc.docevent_set.filter( - Q(type="started_iesg_process") | Q(type="changed_state") - ) - .order_by("-time") - .first() - ) + state_events = doc.docevent_set.filter( + Q(type="started_iesg_process") | Q(type="changed_state"), + # desc__contains="IESG state changed to", + # time__gte=dates[0] + ).order_by("-time") + + last_state_event = state_events.last() + if (last_state_event is not None) and ( right_now - last_state_event.time ) > delta: if state in ad.doc_now[dt]: ad.doc_prev[dt][state].add(doc) + # compute state history for drafts + print() + print(doc) + for e in state_events: + # get state name changed into + match = re.search( + r"state changed to (.*?)(?:::.*)? from (.*?)(?=::|$)", + strip_tags(e.desc), + flags=re.MULTILINE, + ) + if not match: + # some other state change, ignore + continue + + to_state = match[1] + if to_state not in state_slugs[dt]: + # change into a state we don't display, ignore + continue + + for idx, start_date in enumerate(dates): + # print(idx, start_date.date()) + cutoff = right_now if idx == 0 else dates[idx - 1] + if e.time <= cutoff: + # skip if doc is already in the same bucket + if any( + [ + doc.name in ad.buckets[dt][state][idx] + for state, _ in AD_WORKLOAD_STATE_SLUGS[dt] + ] + ): + print( + f"skip {e.time.date()} to bucket {idx} {to_state} (exists) " + ) + continue + print( + f"ADD {e.time.date()} to bucket {idx} {to_state} (<= {cutoff.date()}) " + ) + ad.buckets[dt][state_slugs[dt][to_state]][idx].append(doc.name) + else: + # no need to check earlier buckets (based on date of event) + print( + f"skip {e.time.date()} to buckets >= {idx} {to_state} (> {cutoff.date()}) " + ) + break + for dt in AD_WORKLOAD_STATE_SLUGS: for state in ad.doc_now[dt]: + if dt == "draft" and state == "pub-req": + addn = {n.name for n in ad.doc_now[dt][state]} + print("old", dt, state, len(addn), addn) + print("new", dt, state, len(ad.buckets[dt][state][0]), ad.buckets[dt][state][0]) + print("DIFF", set(ad.buckets[dt][state][0]) ^ addn) ad.doc_diff[dt][state] = ad.doc_prev[dt][state] ^ ad.doc_now[dt][state] + ad.buckets[dt][state].reverse() workload = [ dict( - doc_type=doc_type_name(dt), + doc_type=dt, + doc_type_name=doc_type_name(dt), state_names=[state_name(dt, state) for state in ad.doc_now[dt]], counts=[ ( @@ -441,7 +524,17 @@ def ad_workload(request): for dt in AD_WORKLOAD_STATE_SLUGS ] - return render(request, "doc/ad_list.html", {"workload": workload, "delta": delta}) + return render( + request, + "doc/ad_list.html", + { + "workload": workload, + "delta": weeks, + "data": {dt: {slugify(ad): ad.buckets[dt] for ad in ads} for dt in AD_WORKLOAD_STATE_SLUGS}, + "bucket_cutoffs": [date.date() for date in dates], + }, + ) + def docs_for_ad(request, name): @@ -467,7 +560,6 @@ def sort_key(doc): results, meta = prepare_document_table(request, Document.objects.filter(ad=ad)) results.sort(key=lambda d: sort_key(d)) - del meta["headers"][-1] # filter out some results results = [ diff --git a/ietf/static/css/highcharts.scss b/ietf/static/css/highcharts.scss index c556f4e118..d2f5d5e0e7 100644 --- a/ietf/static/css/highcharts.scss +++ b/ietf/static/css/highcharts.scss @@ -1 +1,6 @@ @import "npm:highcharts/css/highcharts.css"; +@import "custom-bs-import"; + +.highcharts-container { + font-family: $font-family-sans-serif; +} diff --git a/ietf/static/js/highcharts.js b/ietf/static/js/highcharts.js index 268f96e50f..6c3b68051f 100644 --- a/ietf/static/js/highcharts.js +++ b/ietf/static/js/highcharts.js @@ -6,6 +6,8 @@ import Highcharts_Export_Data from "highcharts/modules/export-data"; import Highcharts_Accessibility from "highcharts/modules/accessibility"; import Highcharts_Sunburst from "highcharts/modules/sunburst"; +document.documentElement.style.setProperty("--highcharts-background-color", "transparent"); + Highcharts_Exporting(Highcharts); Highcharts_Offline_Exporting(Highcharts); Highcharts_Export_Data(Highcharts); @@ -27,7 +29,7 @@ window.Highcharts = Highcharts; window.group_stats = function (url, chart_selector) { $.getJSON(url, function (data) { $(chart_selector) - .each(function (i, e) { + .each(function (_, e) { const dataset = e.dataset.dataset; if (!dataset) { console.log("dataset data attribute not set"); diff --git a/ietf/static/js/highstock.js b/ietf/static/js/highstock.js index 15ddb5ef9c..05b1250ed0 100644 --- a/ietf/static/js/highstock.js +++ b/ietf/static/js/highstock.js @@ -5,6 +5,8 @@ import Highcharts_Offline_Exporting from "highcharts/modules/offline-exporting"; import Highcharts_Export_Data from "highcharts/modules/export-data"; import Highcharts_Accessibility from"highcharts/modules/accessibility"; +document.documentElement.style.setProperty("--highcharts-background-color", "transparent"); + Highcharts_Exporting(Highcharts); Highcharts_Offline_Exporting(Highcharts); Highcharts_Export_Data(Highcharts); diff --git a/ietf/templates/doc/ad_count.html b/ietf/templates/doc/ad_count.html index e40baad34d..34b8de70c6 100644 --- a/ietf/templates/doc/ad_count.html +++ b/ietf/templates/doc/ad_count.html @@ -1,6 +1,8 @@ {# Copyright The IETF Trust 2015, All Rights Reserved #} {% load ietf_filters %} +{{ count }} +{% comment %} {% if prev or count %} {{ count }} {% if user|has_role:"Area Director,Secretariat" %} @@ -28,4 +30,5 @@ {% endif %} > {% endif %} -{% endif %} \ No newline at end of file +{% endif %} +{% endcomment %} \ No newline at end of file diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html index 717391a0ce..8336959fbc 100644 --- a/ietf/templates/doc/ad_list.html +++ b/ietf/templates/doc/ad_list.html @@ -3,7 +3,16 @@ {% load origin static %} {% load ietf_filters %} {% block pagehead %} - + + +{% endblock %} +{% block morecss %} + .highcharts-container .highcharts-axis-labels { font-size: .7rem; } + .highcharts-container .highcharts-graph { stroke-width: 2.5; } + .highcharts-container .highcharts-data-label text { + font-size: 1rem; + {# font-weight: inherit; #} + } {% endblock %} {% block title %}IESG Dashboard{% endblock %} {% block content %} @@ -11,19 +20,19 @@

    IESG Dashboard

    {% if user|has_role:"Area Director,Secretariat" %}
    - {{ delta.days }}-day trend indicators + {{ delta }}-week trend graphs are only shown to logged-in Area Directors.
    {% endif %} {% for type in workload %} -

    {{ type.doc_type }} State Counts

    -
    Area Director - {{ g|split:'/'|join:'/' }} + {% for state in type.state_names %} + + {{ state|split:'/'|join:'/' }}
    {{ ad.name }} + id="{{ type.doc_type|slugify }}-{{ ad.full_name_as_key|slugify }}-{{ label.0|slugify }}"> {% include 'doc/ad_count.html' %}
    Sum {% include 'doc/ad_count.html' %} {{ ad.name }} + id="{{ type.doc_type|slugify }}-{{ ad.full_name_as_key|slugify }}-{{ label|slugify }}"> {% include 'doc/ad_count.html' %}
    Sum {% include 'doc/ad_count.html' %}
    +

    {{ type.doc_type_name }} State Counts

    +
    {% for state in type.state_names %} {% endfor %} @@ -35,10 +44,11 @@

    {{ type.doc_type }} State Coun

    - {% for label, up_is_good, count, prev, docs_delta in ad_data %} + {% for state, up_is_good, count, prev, docs_delta in ad_data %} {% endfor %} @@ -47,9 +57,10 @@

    {{ type.doc_type }} State Coun

    - {% for label, up_is_good, count, prev in type.sums %} + {% for state, up_is_good, count, prev in type.sums %} {% endfor %} @@ -69,4 +80,107 @@

    {{ type.doc_type }} State Coun }); }); + + {{ data|json_script:"data" }} + {{ bucket_cutoffs|json_script:"bucket-cutoffs" }} + {% endblock %} \ No newline at end of file From 32586bd9d84ecc02529ed69fc753766ded8a1738 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Fri, 3 Nov 2023 13:32:13 +0200 Subject: [PATCH 13/22] More progress --- ietf/doc/tests.py | 6 +- ietf/doc/utils_search.py | 4 +- ietf/doc/views_search.py | 227 ++++++++++++++++++++------------ ietf/templates/doc/ad_list.html | 4 +- 4 files changed, 153 insertions(+), 88 deletions(-) diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 2ded666af1..6242cc2d71 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -59,7 +59,7 @@ from ietf.utils.test_utils import TestCase from ietf.utils.text import normalize_text from ietf.utils.timezone import date_today, datetime_today, DEADLINE_TZINFO, RPC_TZINFO -from ietf.doc.utils_search import AD_WORKLOAD_STATE_SLUGS +from ietf.doc.utils_search import AD_WORKLOAD class SearchTests(TestCase): @@ -287,8 +287,8 @@ def test_ad_workload(self): person__name="Example Areadirector", ).person expected = defaultdict(lambda: 0) - for doc_type_slug in AD_WORKLOAD_STATE_SLUGS: - for state, _ in AD_WORKLOAD_STATE_SLUGS[doc_type_slug]: + for doc_type_slug in AD_WORKLOAD: + for state, _ in AD_WORKLOAD[doc_type_slug]: target_num = random.randint(0, 2) for _ in range(target_num): if ( diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py index bcf1968eb0..e447bd0b17 100644 --- a/ietf/doc/utils_search.py +++ b/ietf/doc/utils_search.py @@ -293,7 +293,7 @@ def num(i): # "rfc" is a custom subset of "draft" that we special-case in the code # to break out these docs into a separate table. # -AD_WORKLOAD_STATE_SLUGS = { +AD_WORKLOAD = { "draft": [ ("pub-req", False), ("ad-eval", False), @@ -308,7 +308,7 @@ def num(i): ], "rfc": [ ("rfcqueue", True), - ("rfc", True) + ("rfc", True), ], "conflrev": [ ("needshep", False), diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 8aed77c62e..31d41eb3cf 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -67,7 +67,7 @@ from ietf.person.utils import get_active_ads from ietf.utils.draft_search import normalize_draftname from ietf.utils.log import log -from ietf.doc.utils_search import prepare_document_table, doc_type, doc_state, doc_type_name, AD_WORKLOAD_STATE_SLUGS +from ietf.doc.utils_search import prepare_document_table, doc_type, doc_state, doc_type_name, AD_WORKLOAD class SearchForm(forms.Form): @@ -317,12 +317,14 @@ def cached_redirect(cache_key, url): def state_name(doc_type, state, shorten=True): name = "" - if doc_type in ["draft", "rfc"] and state not in ["rfc", "expired"]: + if state == "ignore": + name = state + elif doc_type in ["draft", "rfc"] and state not in ["rfc", "expired"]: name = State.objects.get(type="draft-iesg", slug=state).name elif state == "rfc": name = "RFC" elif doc_type == "conflrev" and state.startswith("appr"): - name = "Approved" + name = "Approved" else: name = State.objects.get(type=doc_type, slug=state).name @@ -359,24 +361,37 @@ def state_name(doc_type, state, shorten=True): return name.strip() +# def slug_to_name(dt, ds): +# if ds == "ignore": +# return ds +# if dt == "rfc" and ds == "rfc": +# return "RFC" +# if dt == "conflrev" and ds == "approved": +# return "Approved" +# if dt == "draft" or ds == "rfcqueue": +# s = State.objects.get(slug=ds, type="draft-iesg") +# else: +# s = State.objects.get(slug=ds, type=dt) +# return s.name + + +STATE_SLUGS = { + dt: {state_name(dt, ds, shorten=False): ds for ds, _ in AD_WORKLOAD[dt] + [("ignore", "ignore")]} + for dt in AD_WORKLOAD +} + +IESG_STATES = State.objects.filter(type="draft-iesg").values_list("name", flat=True) + + +def date_to_bucket(date, now): + # buckets go into the past with increasing index; now.date() is bucket 0 + return int((now.date() - date.date()).total_seconds() / 60 / 60 / 24) + + def ad_workload(request): - weeks = 17 - delta = datetime.timedelta(weeks=weeks) - right_now = timezone.now() - - state_slugs = {} - for dt in AD_WORKLOAD_STATE_SLUGS: - state_slugs[dt] = {} - for ds, _ in AD_WORKLOAD_STATE_SLUGS[dt]: - if dt == "rfc" and ds == "rfc": - state_slugs[dt]["RFC"] = "rfc" - elif dt == "draft" or ds == "rfcqueue": - s = State.objects.get(slug=ds, type="draft-iesg") - elif dt == "conflrev" and ds == "approved": - state_slugs[dt]["Approved"] = ds - else: - s = State.objects.get(slug=ds, type=dt) - state_slugs[dt][s.name] = ds + buckets = 120 + # delta = datetime.timedelta(weeks=weeks) + now = timezone.now() ads = [] responsible = Document.objects.values_list("ad", flat=True).distinct() @@ -389,29 +404,29 @@ def ad_workload(request): | Q(pk__in=responsible) ).distinct(): if p in get_active_ads(): - if p.name == "John Scudder": + if p.name == "Lars Eggert": ads.append(p) - dates = list( - dateutil.rrule.rrule( - freq=dateutil.rrule.WEEKLY, count=13, dtstart=right_now - delta - ) - ) - dates.reverse() + # dates = list( + # dateutil.rrule.rrule( + # freq=dateutil.rrule.WEEKLY, count=13, dtstart=now - delta + # ) + # ) + # dates.reverse() for ad in ads: ad.dashboard = urlreverse( "ietf.doc.views_search.docs_for_ad", kwargs=dict(name=ad.full_name_as_key()) ) ad.doc_now = { - dt: {state: set() for state, _ in AD_WORKLOAD_STATE_SLUGS[dt]} - for dt in AD_WORKLOAD_STATE_SLUGS + dt: {state: set() for state, _ in AD_WORKLOAD[dt]} + for dt in AD_WORKLOAD } ad.doc_prev = copy.deepcopy(ad.doc_now) ad.doc_diff = copy.deepcopy(ad.doc_now) ad.buckets = { - dt: {state: [[] for _ in dates] for state, _ in AD_WORKLOAD_STATE_SLUGS[dt]} - for dt in AD_WORKLOAD_STATE_SLUGS + dt: {state: [[] for _ in range(buckets)] for state in STATE_SLUGS[dt].values()} + for dt in STATE_SLUGS } for doc in Document.objects.filter(ad=ad): @@ -422,71 +437,117 @@ def ad_workload(request): ad.doc_now[dt][state].add(doc) state_events = doc.docevent_set.filter( - Q(type="started_iesg_process") | Q(type="changed_state"), + Q(type="started_iesg_process") | Q(type="changed_state") | Q(type="published_rfc"), # desc__contains="IESG state changed to", # time__gte=dates[0] ).order_by("-time") last_state_event = state_events.last() - if (last_state_event is not None) and ( - right_now - last_state_event.time - ) > delta: - if state in ad.doc_now[dt]: - ad.doc_prev[dt][state].add(doc) + # if (last_state_event is not None) and ( + # now - last_state_event.time + # ) > delta: + # if state in ad.doc_now[dt]: + # ad.doc_prev[dt][state].add(doc) # compute state history for drafts print() print(doc) + last = now for e in state_events: - # get state name changed into + print(e.time.date(), e.desc) + # get the state name this event changed the doc into match = re.search( - r"state changed to (.*?)(?:::.*)? from (.*?)(?=::|$)", + r"(RFC) published|[Ss]tate changed to (.*?)(?:::.*)? from (.*?)(?=::|$)", strip_tags(e.desc), flags=re.MULTILINE, ) if not match: - # some other state change, ignore - continue - - to_state = match[1] - if to_state not in state_slugs[dt]: - # change into a state we don't display, ignore + # some irrelevant state change for the AD dashboard, ignore it continue - for idx, start_date in enumerate(dates): - # print(idx, start_date.date()) - cutoff = right_now if idx == 0 else dates[idx - 1] - if e.time <= cutoff: - # skip if doc is already in the same bucket - if any( - [ - doc.name in ad.buckets[dt][state][idx] - for state, _ in AD_WORKLOAD_STATE_SLUGS[dt] - ] - ): - print( - f"skip {e.time.date()} to bucket {idx} {to_state} (exists) " - ) - continue - print( - f"ADD {e.time.date()} to bucket {idx} {to_state} (<= {cutoff.date()}) " - ) - ad.buckets[dt][state_slugs[dt][to_state]][idx].append(doc.name) - else: - # no need to check earlier buckets (based on date of event) - print( - f"skip {e.time.date()} to buckets >= {idx} {to_state} (> {cutoff.date()}) " - ) + to_state = match.group(1) or match.group(2) + # fix up some things + if dt == "conflrev" and to_state.startswith("Approved"): + to_state = "Approved" + elif to_state == "RFC Published": + to_state = "RFC" + print(to_state, dt, STATE_SLUGS[dt].keys()) + if to_state not in STATE_SLUGS[dt].keys(): + # change into a state the AD dashboard doesn't display + if to_state in IESG_STATES: + # if it's an IESG state we don't display, we're done with this doc + print("iesg state, break") break + # if it's not an IESG state, keep going with next event + # print(dt, state, to_state, STATE_SLUGS[dt].keys()) + print("not shown state, cont") + continue - for dt in AD_WORKLOAD_STATE_SLUGS: + sn = STATE_SLUGS[dt][to_state] + + buckets_start = date_to_bucket(last, now) + buckets_end = date_to_bucket(e.time, now) + if buckets_end >= buckets: + # this event is older than we record in the history + if last == now: + # but since we didn't record any state yet, + # this is the state the doc was in for the + # entire history + print(f"set initial: {dt} {sn}") + for b in range(buckets_start, buckets): + ad.buckets[dt][sn][b].append(doc.name) + break + + # print(last) + # print(e.time) + # print(buckets) + for b in range(buckets_start, buckets_end): + ad.buckets[dt][sn][b].append(doc.name) + + last = e.time + + # for idx, start_date in enumerate(dates): + # # print(idx, start_date.date()) + # cutoff = now if idx == 0 else dates[idx - 1] + # if e.time <= cutoff: + # # skip if doc is already in the same bucket + # if any( + # [ + # doc.name in ad.buckets[dt][state][idx] + # for state in STATE_SLUGS[dt].values() + # ] + # ): + # print( + # f"skip {e.time.date()} to bucket {idx} {to_state} (exists) " + # ) + # continue + # print( + # f"ADD {e.time.date()} to bucket {idx} {to_state} (<= {cutoff.date()}) " + # ) + # ad.buckets[dt][sn][idx].append(doc.name) + # else: + # # no need to check earlier buckets (based on date of event) + # print( + # f"skip {e.time.date()} to buckets >= {idx} {to_state} (> {cutoff.date()}) " + # ) + # break + + for dt in AD_WORKLOAD: for state in ad.doc_now[dt]: - if dt == "draft" and state == "pub-req": - addn = {n.name for n in ad.doc_now[dt][state]} + addn = {n.name for n in ad.doc_now[dt][state]} + new = set(ad.buckets[dt][state][0]) + diff = new ^ addn + if diff: print("old", dt, state, len(addn), addn) - print("new", dt, state, len(ad.buckets[dt][state][0]), ad.buckets[dt][state][0]) - print("DIFF", set(ad.buckets[dt][state][0]) ^ addn) + print( + "new", + dt, + state, + len(new), + new, + ) + print("DIFF", diff) ad.doc_diff[dt][state] = ad.doc_prev[dt][state] ^ ad.doc_now[dt][state] ad.buckets[dt][state].reverse() @@ -501,7 +562,7 @@ def ad_workload(request): [ ( state, - {s: uig for s, uig in AD_WORKLOAD_STATE_SLUGS[dt]}[state], + {s: uig for s, uig in AD_WORKLOAD[dt]}[state], len(ad.doc_now[dt][state]), len(ad.doc_prev[dt][state]), ad.doc_diff[dt][state], @@ -514,14 +575,14 @@ def ad_workload(request): sums=[ ( state, - {s: uig for s, uig in AD_WORKLOAD_STATE_SLUGS[dt]}[state], + {s: uig for s, uig in AD_WORKLOAD[dt]}[state], sum([len(ad.doc_now[dt][state]) for ad in ads]), sum([len(ad.doc_prev[dt][state]) for ad in ads]), ) for state in ad.doc_now[dt] ], ) - for dt in AD_WORKLOAD_STATE_SLUGS + for dt in AD_WORKLOAD ] return render( @@ -529,17 +590,19 @@ def ad_workload(request): "doc/ad_list.html", { "workload": workload, - "delta": weeks, - "data": {dt: {slugify(ad): ad.buckets[dt] for ad in ads} for dt in AD_WORKLOAD_STATE_SLUGS}, - "bucket_cutoffs": [date.date() for date in dates], + # "delta": weeks, + "data": { + dt: {slugify(ad): ad.buckets[dt] for ad in ads} + for dt in AD_WORKLOAD + }, + # "bucket_cutoffs": [date.date() for date in dates], }, ) - def docs_for_ad(request, name): def sort_key(doc): - key = list(AD_WORKLOAD_STATE_SLUGS.keys()).index(doc_type(doc)) + key = list(AD_WORKLOAD.keys()).index(doc_type(doc)) return key ad = None diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html index 8336959fbc..65bf830bb3 100644 --- a/ietf/templates/doc/ad_list.html +++ b/ietf/templates/doc/ad_list.html @@ -107,12 +107,14 @@

    {{ type.doc_type_name }} State Counts< // console.log(state,"aft", sum[state]) }); }); - // console.log(sum) Object.entries(ads).forEach(([ad, states]) => { Object.entries(states).forEach(([state, buckets]) => { const cell = `chart-${dt}-${ad}-${state}`; + if (state == "ignore") + return; + // if (buckets.series.every((x) => x == 0)) { // document.getElementById(cell).innerHTML = '
    0
    '; // return; From d04c582bbc7feec85c3043442c9d9bfd329fa23c Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Fri, 3 Nov 2023 17:31:38 +0200 Subject: [PATCH 14/22] Getting close --- ietf/doc/utils_search.py | 6 +- ietf/doc/views_search.py | 191 ++++++++++++++----------------- ietf/templates/doc/ad_count.html | 34 ------ ietf/templates/doc/ad_list.html | 33 ++---- 4 files changed, 95 insertions(+), 169 deletions(-) delete mode 100644 ietf/templates/doc/ad_count.html diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py index e447bd0b17..4246f0ac84 100644 --- a/ietf/doc/utils_search.py +++ b/ietf/doc/utils_search.py @@ -315,7 +315,7 @@ def num(i): ("adrev", False), ("iesgeval", True), ("approved", True), # synthesized state for all the "appr-" states - ("withdraw", None), + # ("withdraw", None), # probably not a useful state to show ], "statchg": [ ("needshep", False), @@ -325,7 +325,7 @@ def num(i): ("iesgeval", True), ("goahead", False), ("appr-sent", True), - ("dead", None), + # ("dead", None), # probably not a useful state to show ], "charter": [ ("notrev", None), @@ -334,7 +334,7 @@ def num(i): ("extrev", True), ("iesgrev", True), ("approved", True), - ("replaced", None), + # ("replaced", None), # probably not a useful state to show ], } diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 31d41eb3cf..d3566c51b6 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -37,7 +37,6 @@ import re import datetime import copy -import dateutil.rrule from django import forms from django.conf import settings @@ -317,10 +316,8 @@ def cached_redirect(cache_key, url): def state_name(doc_type, state, shorten=True): name = "" - if state == "ignore": - name = state - elif doc_type in ["draft", "rfc"] and state not in ["rfc", "expired"]: - name = State.objects.get(type="draft-iesg", slug=state).name + if doc_type in ["draft", "rfc"] and state not in ["rfc", "expired"]: + name = State.objects.get(type__in=["draft", "draft-iesg"], slug=state).name elif state == "rfc": name = "RFC" elif doc_type == "conflrev" and state.startswith("appr"): @@ -361,22 +358,8 @@ def state_name(doc_type, state, shorten=True): return name.strip() -# def slug_to_name(dt, ds): -# if ds == "ignore": -# return ds -# if dt == "rfc" and ds == "rfc": -# return "RFC" -# if dt == "conflrev" and ds == "approved": -# return "Approved" -# if dt == "draft" or ds == "rfcqueue": -# s = State.objects.get(slug=ds, type="draft-iesg") -# else: -# s = State.objects.get(slug=ds, type=dt) -# return s.name - - STATE_SLUGS = { - dt: {state_name(dt, ds, shorten=False): ds for ds, _ in AD_WORKLOAD[dt] + [("ignore", "ignore")]} + dt: {state_name(dt, ds, shorten=False): ds for ds, _ in AD_WORKLOAD[dt]} for dt in AD_WORKLOAD } @@ -389,8 +372,7 @@ def date_to_bucket(date, now): def ad_workload(request): - buckets = 120 - # delta = datetime.timedelta(weeks=weeks) + days = 120 now = timezone.now() ads = [] @@ -404,30 +386,24 @@ def ad_workload(request): | Q(pk__in=responsible) ).distinct(): if p in get_active_ads(): - if p.name == "Lars Eggert": - ads.append(p) + ads.append(p) - # dates = list( - # dateutil.rrule.rrule( - # freq=dateutil.rrule.WEEKLY, count=13, dtstart=now - delta - # ) - # ) - # dates.reverse() + bucket_template = { + dt: {state: [[] for _ in range(days)] for state in STATE_SLUGS[dt].values()} + for dt in STATE_SLUGS + } + sums = copy.deepcopy(bucket_template) for ad in ads: ad.dashboard = urlreverse( "ietf.doc.views_search.docs_for_ad", kwargs=dict(name=ad.full_name_as_key()) ) ad.doc_now = { - dt: {state: set() for state, _ in AD_WORKLOAD[dt]} - for dt in AD_WORKLOAD + dt: {state: set() for state, _ in AD_WORKLOAD[dt]} for dt in AD_WORKLOAD } ad.doc_prev = copy.deepcopy(ad.doc_now) ad.doc_diff = copy.deepcopy(ad.doc_now) - ad.buckets = { - dt: {state: [[] for _ in range(buckets)] for state in STATE_SLUGS[dt].values()} - for dt in STATE_SLUGS - } + ad.buckets = copy.deepcopy(bucket_template) for doc in Document.objects.filter(ad=ad): dt = doc_type(doc) @@ -437,101 +413,96 @@ def ad_workload(request): ad.doc_now[dt][state].add(doc) state_events = doc.docevent_set.filter( - Q(type="started_iesg_process") | Q(type="changed_state") | Q(type="published_rfc"), - # desc__contains="IESG state changed to", - # time__gte=dates[0] + Q(type="started_iesg_process") + | Q(type="changed_state") + | Q(type="published_rfc") + | Q(type="closed_ballot"), ).order_by("-time") - last_state_event = state_events.last() - - # if (last_state_event is not None) and ( - # now - last_state_event.time - # ) > delta: - # if state in ad.doc_now[dt]: - # ad.doc_prev[dt][state].add(doc) - # compute state history for drafts - print() - print(doc) + # print() + # print(doc) last = now + ballot_closed = False for e in state_events: - print(e.time.date(), e.desc) - # get the state name this event changed the doc into - match = re.search( - r"(RFC) published|[Ss]tate changed to (.*?)(?:::.*)? from (.*?)(?=::|$)", - strip_tags(e.desc), - flags=re.MULTILINE, - ) - if not match: - # some irrelevant state change for the AD dashboard, ignore it - continue + # print(e.time.date(), e.type, e.desc) + to_state = None + if dt == "charter": + if e.type == "closed_ballot": + to_state = state_name(dt, state, shorten=False) + elif e.desc.endswith("has been replaced"): + # stop tracking + break - to_state = match.group(1) or match.group(2) - # fix up some things + if not to_state: + # get the state name this event changed the doc into + match = re.search( + r"(RFC) published|[Ss]tate changed to (.*?)(?:::.*)? from (.*?)(?=::|$)", + strip_tags(e.desc), + flags=re.MULTILINE, + ) + if not match: + # some irrelevant state change for the AD dashboard, ignore it + continue + to_state = match.group(1) or match.group(2) + + # fix up some states that have been renamed if dt == "conflrev" and to_state.startswith("Approved"): to_state = "Approved" + elif dt == "charter" and to_state.startswith( + "Start Chartering/Rechartering" + ): + to_state = "Start Chartering/Rechartering (Internal Steering Group/IAB Review)" elif to_state == "RFC Published": to_state = "RFC" - print(to_state, dt, STATE_SLUGS[dt].keys()) - if to_state not in STATE_SLUGS[dt].keys(): + # print(to_state, dt, STATE_SLUGS[dt].keys()) + + if to_state not in STATE_SLUGS[dt].keys() or to_state == "Replaced": # change into a state the AD dashboard doesn't display - if to_state in IESG_STATES: + if to_state in IESG_STATES or to_state == "Replaced": # if it's an IESG state we don't display, we're done with this doc - print("iesg state, break") + # print("iesg state, break") + last = e.time break # if it's not an IESG state, keep going with next event # print(dt, state, to_state, STATE_SLUGS[dt].keys()) - print("not shown state, cont") + # print("not shown state, cont") continue sn = STATE_SLUGS[dt][to_state] buckets_start = date_to_bucket(last, now) buckets_end = date_to_bucket(e.time, now) - if buckets_end >= buckets: + if buckets_end >= days: # this event is older than we record in the history if last == now: # but since we didn't record any state yet, # this is the state the doc was in for the # entire history - print(f"set initial: {dt} {sn}") - for b in range(buckets_start, buckets): + # print(f"set initial: {dt} {sn}") + for b in range(buckets_start, days): ad.buckets[dt][sn][b].append(doc.name) + sums[dt][sn][b].append(doc.name) + last = e.time break - # print(last) - # print(e.time) - # print(buckets) + # record doc state in the indicated buckets for b in range(buckets_start, buckets_end): ad.buckets[dt][sn][b].append(doc.name) - + sums[dt][sn][b].append(doc.name) last = e.time - # for idx, start_date in enumerate(dates): - # # print(idx, start_date.date()) - # cutoff = now if idx == 0 else dates[idx - 1] - # if e.time <= cutoff: - # # skip if doc is already in the same bucket - # if any( - # [ - # doc.name in ad.buckets[dt][state][idx] - # for state in STATE_SLUGS[dt].values() - # ] - # ): - # print( - # f"skip {e.time.date()} to bucket {idx} {to_state} (exists) " - # ) - # continue - # print( - # f"ADD {e.time.date()} to bucket {idx} {to_state} (<= {cutoff.date()}) " - # ) - # ad.buckets[dt][sn][idx].append(doc.name) - # else: - # # no need to check earlier buckets (based on date of event) - # print( - # f"skip {e.time.date()} to buckets >= {idx} {to_state} (> {cutoff.date()}) " - # ) - # break + if last == now: + s = state_name(dt, state, shorten=False) + # print(dt, s, state) + if s in STATE_SLUGS[dt].keys(): + # we didn't have a single event for this doc, assume + # the current state applied throughput the history + for b in range(days): + ad.buckets[dt][state][b].append(doc.name) + sums[dt][state][b].append(doc.name) + # else: + # print(STATE_SLUGS[dt].keys()) for dt in AD_WORKLOAD: for state in ad.doc_now[dt]: @@ -549,7 +520,6 @@ def ad_workload(request): ) print("DIFF", diff) ad.doc_diff[dt][state] = ad.doc_prev[dt][state] ^ ad.doc_now[dt][state] - ad.buckets[dt][state].reverse() workload = [ dict( @@ -585,21 +555,26 @@ def ad_workload(request): for dt in AD_WORKLOAD ] + data = {dt: {slugify(ad): ad.buckets[dt] for ad in ads} for dt in AD_WORKLOAD} + + for ad in ads: + for dt in AD_WORKLOAD: + for state in sums[dt]: + ad.buckets[dt][state].reverse() + + for dt in AD_WORKLOAD: + for state in sums[dt]: + sums[dt][state].reverse() + data[dt]["sum"] = sums[dt] + return render( request, "doc/ad_list.html", - { - "workload": workload, - # "delta": weeks, - "data": { - dt: {slugify(ad): ad.buckets[dt] for ad in ads} - for dt in AD_WORKLOAD - }, - # "bucket_cutoffs": [date.date() for date in dates], - }, + {"workload": workload, "delta": days, "data": data}, ) + def docs_for_ad(request, name): def sort_key(doc): key = list(AD_WORKLOAD.keys()).index(doc_type(doc)) diff --git a/ietf/templates/doc/ad_count.html b/ietf/templates/doc/ad_count.html deleted file mode 100644 index 34b8de70c6..0000000000 --- a/ietf/templates/doc/ad_count.html +++ /dev/null @@ -1,34 +0,0 @@ -{# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load ietf_filters %} - -{{ count }} -{% comment %} -{% if prev or count %} - {{ count }} - {% if user|has_role:"Area Director,Secretariat" %} - {{ d.name }} - {% endfor %} - - {% endif %}" - {% endif %} - {% if prev < count %} - class="bi bi-arrow-up-right-circle{% if count %}-fill{% endif %} {{ up_is_good|yesno:'text-success,text-danger,text-body-secondary' }}" - {% elif prev > count %} - class="bi bi-arrow-down-right-circle{% if count %}-fill{% endif %} {{ up_is_good|yesno:'text-danger,text-success,text-body-secondary' }}" - {% else %} - class="bi bi-arrow-right-circle text-body-secondary" - {% endif %} - > - {% endif %} -{% endif %} -{% endcomment %} \ No newline at end of file diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html index 65bf830bb3..cfc4eedc9a 100644 --- a/ietf/templates/doc/ad_list.html +++ b/ietf/templates/doc/ad_list.html @@ -32,7 +32,7 @@

    {{ type.doc_type_name }} State Counts<

    {% for state in type.state_names %} {% endfor %} @@ -47,7 +47,7 @@

    {{ type.doc_type_name }} State Counts< {% for state, up_is_good, count, prev, docs_delta in ad_data %}

    {% endfor %} @@ -59,7 +59,7 @@

    {{ type.doc_type_name }} State Counts<

    {% for state, up_is_good, count, prev in type.sums %} {% endfor %} @@ -82,29 +82,17 @@

    {{ type.doc_type_name }} State Counts< {{ data|json_script:"data" }} - {{ bucket_cutoffs|json_script:"bucket-cutoffs" }} {% endblock %} \ No newline at end of file From 12aca4f629598498fb7057a070617d5ed9fd9f4f Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Sat, 4 Nov 2023 14:21:17 +0100 Subject: [PATCH 16/22] Remove unused variable --- ietf/doc/views_search.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 7614561545..1058e9a13d 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -416,7 +416,6 @@ def ad_workload(request): # compute state history for drafts last = now - ballot_closed = False for e in state_events: to_state = None if dt == "charter": From 1219fa097669fe375e9fee98916488d7ee843756 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Sat, 4 Nov 2023 15:42:13 +0100 Subject: [PATCH 17/22] Suppress mypy warning --- ietf/doc/views_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 1058e9a13d..66d1ad1b22 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -360,7 +360,7 @@ def state_name(doc_type, state, shorten=True): STATE_SLUGS = { - dt: {state_name(dt, ds, shorten=False): ds for ds, _ in AD_WORKLOAD[dt]} + dt: {state_name(dt, ds, shorten=False): ds for ds, _ in AD_WORKLOAD[dt]} # type: ignore for dt in AD_WORKLOAD } From 4e3591633356eb8da08ac061cc73c783526813f6 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Sat, 4 Nov 2023 16:28:15 +0100 Subject: [PATCH 18/22] Fix #6553 --- ietf/doc/views_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 66d1ad1b22..5499b32659 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -614,7 +614,7 @@ def sort_key(doc): not ballot or doc.get_state_slug("draft") == "repl" or doc.get_state_slug("draft-iesg") == "defer" - or (doc.telechat_date() and doc.telechat_date() > timezone.now().date()) + or not doc.previous_telechat_date() ): continue From 25c44e41b4d7d588e4ae36eb46cc8d548bf9c2e0 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Sun, 5 Nov 2023 15:04:16 +0100 Subject: [PATCH 19/22] Log in as secretary to execute new code, and remove redundant check --- ietf/doc/tests.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 6242cc2d71..7575e4e08d 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -324,21 +324,11 @@ def test_ad_workload(self): type_id=doc_type_slug, slug=state ), ) - + self.login('secretary') url = urlreverse("ietf.doc.views_search.ad_workload") r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - for group_type, ad, group in expected: - self.assertEqual( - int(q(f"#{group_type}-{ad}-{group}").text()), - expected[(group_type, ad, group)], - ) - - url = urlreverse('ietf.doc.views_search.ad_workload') - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) for group_type, ad, group in expected: self.assertEqual(int(q(f'#{group_type}-{ad}-{group}').text()),expected[(group_type, ad, group)]) From d083c740f4bc0ca4eb886f74df92cd499b04361f Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Sun, 5 Nov 2023 15:14:53 +0100 Subject: [PATCH 20/22] Remove unneeded code --- ietf/doc/tests.py | 19 ++++++----- ietf/doc/utils_search.py | 72 +++++++++++++++++++--------------------- ietf/doc/views_search.py | 2 +- 3 files changed, 47 insertions(+), 46 deletions(-) diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 7575e4e08d..ace55a0d79 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -288,7 +288,7 @@ def test_ad_workload(self): ).person expected = defaultdict(lambda: 0) for doc_type_slug in AD_WORKLOAD: - for state, _ in AD_WORKLOAD[doc_type_slug]: + for state in AD_WORKLOAD[doc_type_slug]: target_num = random.randint(0, 2) for _ in range(target_num): if ( @@ -296,7 +296,7 @@ def test_ad_workload(self): or doc_type_slug == "rfc" and state == "rfcqueue" ): - IndividualDraftFactory( + IndividualDraftFactory( ad=ad, states=[ ("draft-iesg", state), @@ -304,33 +304,36 @@ def test_ad_workload(self): ], ) elif doc_type_slug == "rfc": - WgRfcFactory.create( + WgRfcFactory.create( states=[("draft", "rfc"), ("draft-iesg", "pub")] ) elif doc_type_slug == "charter": - CharterFactory(ad=ad, states=[(doc_type_slug, state)]) + CharterFactory(ad=ad, states=[(doc_type_slug, state)]) elif doc_type_slug == "conflrev": - ConflictReviewFactory( + ConflictReviewFactory( ad=ad, states=State.objects.filter( type_id=doc_type_slug, slug=state ), ) elif doc_type_slug == "statchg": - StatusChangeFactory( + StatusChangeFactory( ad=ad, states=State.objects.filter( type_id=doc_type_slug, slug=state ), ) - self.login('secretary') + self.client.login(username="ad", password="ad+password") url = urlreverse("ietf.doc.views_search.ad_workload") r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) for group_type, ad, group in expected: - self.assertEqual(int(q(f'#{group_type}-{ad}-{group}').text()),expected[(group_type, ad, group)]) + self.assertEqual( + int(q(f"#{group_type}-{ad}-{group}").text()), + expected[(group_type, ad, group)], + ) def test_docs_for_ad(self): ad = RoleFactory(name_id='ad',group__type_id='area',group__state_id='active').person diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py index 4246f0ac84..0c2dafd166 100644 --- a/ietf/doc/utils_search.py +++ b/ietf/doc/utils_search.py @@ -26,7 +26,7 @@ def fill_in_telechat_date(docs, doc_dict=None, doc_ids=None): doc_dict = dict((d.pk, d) for d in docs) doc_ids = list(doc_dict.keys()) if doc_ids is None: - doc_ids = list(doc_dict.keys()) + doc_ids = list(doc_dict.keys()) seen = set() for e in TelechatDocEvent.objects.filter(doc__id__in=doc_ids, type="scheduled_for_telechat").order_by('-time'): @@ -181,7 +181,7 @@ def augment_docs_with_related_docs_info(docs): continue originalDoc = d.related_that_doc('conflrev')[0].document d.pages = originalDoc.pages - + def prepare_document_table(request, docs, query=None, max_results=200): """Take a queryset of documents and a QueryDict with sorting info and return list of documents with attributes filled in for @@ -286,55 +286,53 @@ def num(i): # The document types and state slugs to include in the AD dashboard -# and AD doc list, in the order they should be shown. The Boolean -# indicates whether an upwards trend (compared to a past point in -# time) should be considered a positive development or not. +# and AD doc list, in the order they should be shown. # # "rfc" is a custom subset of "draft" that we special-case in the code # to break out these docs into a separate table. # AD_WORKLOAD = { "draft": [ - ("pub-req", False), - ("ad-eval", False), - ("lc-req", True), - ("lc", True), - ("writeupw", False), - # ("defer", False), # probably not a useful state to show, since it's rare - ("iesg-eva", True), - ("goaheadw", False), - ("approved", True), - ("ann", True), + "pub-req", + "ad-eval", + "lc-req", + "lc", + "writeupw", + # "defer", # probably not a useful state to show, since it's rare + "iesg-eva", + "goaheadw", + "approved", + "ann", ], "rfc": [ - ("rfcqueue", True), - ("rfc", True), + "rfcqueue", + "rfc", ], "conflrev": [ - ("needshep", False), - ("adrev", False), - ("iesgeval", True), - ("approved", True), # synthesized state for all the "appr-" states - # ("withdraw", None), # probably not a useful state to show + "needshep", + "adrev", + "iesgeval", + "approved", # synthesized state for all the "appr-" states + # "withdraw", # probably not a useful state to show ], "statchg": [ - ("needshep", False), - ("adrev", False), - ("lc-req", True), - ("in-lc", True), - ("iesgeval", True), - ("goahead", False), - ("appr-sent", True), - # ("dead", None), # probably not a useful state to show + "needshep", + "adrev", + "lc-req", + "in-lc", + "iesgeval", + "goahead", + "appr-sent", + # "dead", # probably not a useful state to show ], "charter": [ - ("notrev", None), - ("infrev", None), - ("intrev", False), - ("extrev", True), - ("iesgrev", True), - ("approved", True), - # ("replaced", None), # probably not a useful state to show + "notrev", + "infrev", + "intrev", + "extrev", + "iesgrev", + "approved", + # "replaced", # probably not a useful state to show ], } diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 5499b32659..2297da3ba9 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -360,7 +360,7 @@ def state_name(doc_type, state, shorten=True): STATE_SLUGS = { - dt: {state_name(dt, ds, shorten=False): ds for ds, _ in AD_WORKLOAD[dt]} # type: ignore + dt: {state_name(dt, ds, shorten=False): ds for ds in AD_WORKLOAD[dt]} # type: ignore for dt in AD_WORKLOAD } From 6b35f4132eb30307d41bee26d2113cd7202f7ee5 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Mon, 6 Nov 2023 09:56:20 +0100 Subject: [PATCH 21/22] Fix #6608 by adding link to state description to state heading --- ietf/doc/views_search.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 2297da3ba9..6ba8abe1e0 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -489,10 +489,8 @@ def ad_workload(request): metadata = [ { - "doc_type": dt, - "doc_type_name": doc_type_name(dt), - "states": ad.buckets[dt].keys(), - "state_names": [state_name(dt, state) for state in ad.buckets[dt]], + "type": (dt, doc_type_name(dt)), + "states": [(state, state_name(dt, state)) for state in ad.buckets[dt]], "ads": ads, } for dt in AD_WORKLOAD From 64f331847c16d53aad6154f7153e7e7fd2bc94db Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Mon, 6 Nov 2023 10:59:06 +0100 Subject: [PATCH 22/22] Missed part of this change in last commit. Also fix an unrelated template nit while I'm here. --- ietf/templates/doc/ad_list.html | 26 ++++++++++++++------------ ietf/templates/help/states.html | 1 - 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html index 03deaa84f8..cfc8830e50 100644 --- a/ietf/templates/doc/ad_list.html +++ b/ietf/templates/doc/ad_list.html @@ -29,30 +29,32 @@

    IESG Dashboard

    are only shown to logged-in Area Directors. {% endif %} - {% for type in metadata %} -

    {{ type.doc_type_name }} State Counts

    + {% for dt in metadata %} +

    {{ dt.type.1 }} State Counts

    Area Director + data-sort="{{ state }}-num"> {{ state|split:'/'|join:'/' }} {{ ad.name }} + id="{{ type.doc_type }}-{{ ad|slugify }}-{{ state }}"> {% include 'doc/ad_count.html' %} +
    Sum {% include 'doc/ad_count.html' %} +
    Area Director + data-sort="{{ state|slugify }}-num"> {{ state|split:'/'|join:'/' }} - {% include 'doc/ad_count.html' %} + {# {{ count }} #}
    		Sum - {% include 'doc/ad_count.html' %} + {# {{ count }} #}
    - {% for state in type.state_names %} + {% for state, state_name in dt.states %} {% endfor %} - {% for ad in type.ads %} + {% for ad in dt.ads %} - {% for state in type.states %} + {% for state, state_name in dt.states %} {% endfor %} @@ -61,9 +63,9 @@

    {{ type.doc_type_name }} State Counts<

    - {% for state in type.states %} + {% for state, state_name in dt.states %} {% endfor %} @@ -139,7 +141,7 @@

    {{ type.doc_type_name }} State Counts< renderTo: cell, panning: { enabled: false }, spacing: [4, 0, 5, 0], - height: "70%", + height: "45%", }, scrollbar: { enabled: false }, tooltip: { enabled: false }, diff --git a/ietf/templates/help/states.html b/ietf/templates/help/states.html index a43bc3a9f5..ccc052647c 100644 --- a/ietf/templates/help/states.html +++ b/ietf/templates/help/states.html @@ -1,7 +1,6 @@ {% extends "base.html" %} {# Copyright The IETF Trust 2015, All Rights Reserved #} {% load origin static ietf_filters textfilters %} -%} {% block pagehead %} {% endblock %}

    Area Director - {{ state|split:'/'|join:'/' }} + data-sort="{{ state }}-num"> + + {{ state_name|split:'/'|join:'/' }} +
    {{ ad.name }} -
    + id="{{ dt.type.0 }}-{{ ad|slugify }}-{{ state }}"> +
    Sum -
    +