Add reviews to search results and IESG agenda. Support restricting

review types so that teams will only have those review types
listed and suggested that they are configured to have.

Fix a couple of things in the importer after having tested it on all
the databases. Set unavailable end date >= 2020 to indefinite.

Fix a couple of bugs.
 - Legacy-Id: 12095

Author: OleLaursen

ietf/doc/resources.py

@@ -516,13 +516,12 @@ class Meta:
 
 
 from ietf.person.resources import PersonResource
-from ietf.review.resources import ReviewRequestResource
 from ietf.name.resources import ReviewRequestStateNameResource
 class ReviewRequestDocEventResource(ModelResource):
     by               = ToOneField(PersonResource, 'by')
     doc              = ToOneField(DocumentResource, 'doc')
     docevent_ptr     = ToOneField(DocEventResource, 'docevent_ptr')
-    review_request   = ToOneField(ReviewRequestResource, 'review_request')
+    review_request   = ToOneField('review.ReviewRequestResource', 'review_request')
     state            = ToOneField(ReviewRequestStateNameResource, 'state', null=True)
     class Meta:
         queryset = ReviewRequestDocEvent.objects.all()

ietf/doc/tests_review.py

@@ -12,7 +12,7 @@
 
 import debug                            # pyflakes:ignore
 
-from ietf.review.models import (ReviewRequest, ReviewTeamResult, ReviewerSettings,
+from ietf.review.models import (ReviewRequest, ResultUsedInReviewTeam, ReviewerSettings,
                                 ReviewWish, UnavailablePeriod, NextReviewerInTeam)
 from ietf.review.utils import reviewer_rotation_list, possibly_advance_next_reviewer_for_team
 import ietf.review.mailarch
@@ -470,7 +470,7 @@ def setup_complete_review_test(self):
         review_req.state = ReviewRequestStateName.objects.get(slug="accepted")
         review_req.save()
         for r in ReviewResultName.objects.filter(slug__in=("issues", "ready")):
-            ReviewTeamResult.objects.get_or_create(team=review_req.team, result=r)
+            ResultUsedInReviewTeam.objects.get_or_create(team=review_req.team, result=r)
         review_req.team.save()
 
         url = urlreverse('ietf.doc.views_review.complete_review', kwargs={ "name": doc.name, "request_id": review_req.pk })
@@ -508,7 +508,7 @@ def test_complete_review_upload_content(self):
         test_file.name = "unnamed"
 
         r = self.client.post(url, data={
-            "result": ReviewResultName.objects.get(reviewteamresult__team=review_req.team, slug="ready").pk,
+            "result": ReviewResultName.objects.get(resultusedinreviewteam__team=review_req.team, slug="ready").pk,
             "state": ReviewRequestStateName.objects.get(slug="completed").pk,
             "reviewed_rev": review_req.doc.rev,
             "review_submission": "upload",
@@ -552,7 +552,7 @@ def test_complete_review_enter_content(self):
         empty_outbox()
 
         r = self.client.post(url, data={
-            "result": ReviewResultName.objects.get(reviewteamresult__team=review_req.team, slug="ready").pk,
+            "result": ReviewResultName.objects.get(resultusedinreviewteam__team=review_req.team, slug="ready").pk,
             "state": ReviewRequestStateName.objects.get(slug="completed").pk,
             "reviewed_rev": review_req.doc.rev,
             "review_submission": "enter",
@@ -583,7 +583,7 @@ def test_complete_review_link_to_mailing_list(self):
         empty_outbox()
 
         r = self.client.post(url, data={
-            "result": ReviewResultName.objects.get(reviewteamresult__team=review_req.team, slug="ready").pk,
+            "result": ReviewResultName.objects.get(resultusedinreviewteam__team=review_req.team, slug="ready").pk,
             "state": ReviewRequestStateName.objects.get(slug="completed").pk,
             "reviewed_rev": review_req.doc.rev,
             "review_submission": "link",
@@ -611,7 +611,7 @@ def test_partially_complete_review(self):
         empty_outbox()
 
         r = self.client.post(url, data={
-            "result": ReviewResultName.objects.get(reviewteamresult__team=review_req.team, slug="ready").pk,
+            "result": ReviewResultName.objects.get(resultusedinreviewteam__team=review_req.team, slug="ready").pk,
             "state": ReviewRequestStateName.objects.get(slug="part-completed").pk,
             "reviewed_rev": review_req.doc.rev,
             "review_submission": "enter",
@@ -645,7 +645,7 @@ def test_partially_complete_review(self):
         url = urlreverse('ietf.doc.views_review.complete_review', kwargs={ "name": review_req.doc.name, "request_id": review_req.pk })
 
         r = self.client.post(url, data={
-            "result": ReviewResultName.objects.get(reviewteamresult__team=review_req.team, slug="ready").pk,
+            "result": ReviewResultName.objects.get(resultusedinreviewteam__team=review_req.team, slug="ready").pk,
             "state": ReviewRequestStateName.objects.get(slug="completed").pk,
             "reviewed_rev": review_req.doc.rev,
             "review_submission": "enter",

ietf/doc/utils_search.py

@@ -60,8 +60,9 @@ def fill_in_document_table_attributes(docs):
         d.expirable = expirable_draft(d)
 
         if d.get_state_slug() != "rfc":
-            d.milestones = d.groupmilestone_set.filter(state="active").order_by("time").select_related("group")
+            d.milestones = sorted((m for m in d.groupmilestone_set.all() if m.state_id == "active"), key=lambda m: m.time)
 
+            d.reviewed_by_teams = sorted(set(r.team for r in d.reviewrequest_set.all()), key=lambda g: g.acronym)
 
 
     # RFCs
@@ -101,7 +102,7 @@ def prepare_document_table(request, docs, query=None, max_results=500):
         # evaluate and fill in attribute results immediately to decrease
         # the number of queries
         docs = docs.select_related("ad", "ad__person", "std_level", "intended_std_level", "group", "stream")
-        docs = docs.prefetch_related("states__type", "tags")
+        docs = docs.prefetch_related("states__type", "tags", "groupmilestone_set__group", "reviewrequest_set__team")
 
     docs = list(docs[:max_results])
 

ietf/doc/views_doc.py

@@ -64,7 +64,7 @@
 from ietf.meeting.models import Session
 from ietf.meeting.utils import group_sessions, get_upcoming_manageable_sessions, sort_sessions
 from ietf.review.models import ReviewRequest
-from ietf.review.utils import can_request_review_of_doc, review_requests_to_list_for_doc
+from ietf.review.utils import can_request_review_of_doc, review_requests_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
 
 def render_document_top(request, doc, tab, name):
@@ -360,7 +360,7 @@ def document_main(request, name, rev=None):
         published = doc.latest_event(type="published_rfc")
         started_iesg_process = doc.latest_event(type="started_iesg_process")
 
-        review_requests = review_requests_to_list_for_doc(doc)
+        review_requests = review_requests_to_list_for_docs([doc]).get(doc.pk, [])
         no_review_from_teams = no_review_from_teams_on_doc(doc, rev or doc.rev)
 
         return render_to_response("doc/document_draft.html",
@@ -586,7 +586,7 @@ def document_main(request, name, rev=None):
 
         other_reviews = []
         if review_req:
-            other_reviews = [r for r in review_requests_to_list_for_doc(review_req.doc) if r != review_req]
+            other_reviews = [r for r in review_requests_to_list_for_docs([review_req.doc]).get(doc.pk, []) if r != review_req]
 
         return render(request, "doc/document_review.html",
                       dict(doc=doc,

ietf/doc/views_review.py

@@ -12,7 +12,7 @@
 from ietf.doc.models import (Document, NewRevisionDocEvent, State, DocAlias,
                              LastCallDocEvent, ReviewRequestDocEvent)
 from ietf.name.models import ReviewRequestStateName, ReviewResultName, DocTypeName
-from ietf.review.models import ReviewRequest
+from ietf.review.models import ReviewRequest, TypeUsedInReviewTeam
 from ietf.group.models import Group
 from ietf.person.fields import PersonEmailChoiceField, SearchablePersonField
 from ietf.ietfauth.utils import is_authorized_in_doc_stream, user_is_person, has_role
@@ -49,16 +49,13 @@ def __init__(self, user, doc, *args, **kwargs):
 
         self.doc = doc
 
-        self.fields['type'].queryset = self.fields['type'].queryset.filter(used=True)
-        self.fields['type'].widget = forms.RadioSelect(choices=[t for t in self.fields['type'].choices if t[0]])
-
         f = self.fields["team"]
         f.queryset = active_review_teams()
-        if not is_authorized_in_doc_stream(user, doc): # user is a reviewer
-            f.queryset = f.queryset.filter(role__name="reviewer", role__person__user=user)
-
         f.initial = [group.pk for group in f.queryset if can_manage_review_requests_for_team(user, group, allow_non_team_personnel=False)]
 
+        self.fields['type'].queryset = self.fields['type'].queryset.filter(used=True, typeusedinreviewteam__team__in=self.fields["team"].queryset).distinct()
+        self.fields['type'].widget = forms.RadioSelect(choices=[t for t in self.fields['type'].choices if t[0]])
+
         self.fields["requested_rev"].label = "Document revision"
 
         if has_role(user, "Secretariat"):
@@ -76,6 +73,17 @@ def clean_deadline(self):
     def clean_requested_rev(self):
         return clean_doc_revision(self.doc, self.cleaned_data.get("requested_rev"))
 
+    def clean(self):
+        chosen_type = self.cleaned_data.get("type")
+        chosen_teams = self.cleaned_data.get("team")
+
+        if chosen_type and chosen_teams:
+            for t in chosen_teams:
+                if not TypeUsedInReviewTeam.objects.filter(type=chosen_type, team=t).exists():
+                    self.add_error("type", "{} does not use the review type {}.".format(t.name, chosen_type.name))
+
+        return self.cleaned_data
+
 @login_required
 def request_review(request, name):
     doc = get_object_or_404(Document, name=name)
@@ -343,7 +351,7 @@ def __init__(self, review_req, *args, **kwargs):
             " ".join("<a class=\"rev label label-default\">{}</a>".format(r)
                      for r in known_revisions))
 
-        self.fields["result"].queryset = self.fields["result"].queryset.filter(reviewteamresult__team=review_req.team)
+        self.fields["result"].queryset = self.fields["result"].queryset.filter(resultusedinreviewteam__team=review_req.team)
         self.fields["review_submission"].choices = [
             (k, label.format(mailing_list=review_req.team.list_email or "[error: team has no mailing list set]"))
             for k, label in self.fields["review_submission"].choices

ietf/group/views_review.py

@@ -10,7 +10,7 @@
 
 from ietf.review.models import ReviewRequest, ReviewerSettings, UnavailablePeriod
 from ietf.review.utils import (can_manage_review_requests_for_team, close_review_request_states,
-                               extract_revision_ordered_review_requests_for_documents,
+                               extract_revision_ordered_review_requests_for_documents_and_replaced,
                                assign_review_request_to_reviewer,
                                close_review_request,
                                setup_reviewer_field,
@@ -201,7 +201,7 @@ def manage_review_requests(request, acronym, group_type=None):
 
     review_requests = suggested_review_requests_for_team(group) + open_review_requests
 
-    document_requests = extract_revision_ordered_review_requests_for_documents(
+    document_requests = extract_revision_ordered_review_requests_for_documents_and_replaced(
         ReviewRequest.objects.filter(state__in=("part-completed", "completed"), team=group).prefetch_related("result"),
         set(r.doc_id for r in review_requests),
     )

ietf/iesg/agenda.py

@@ -9,7 +9,7 @@
 
 from ietf.doc.models import Document, TelechatDocEvent, LastCallDocEvent, ConsensusDocEvent
 from ietf.iesg.models import TelechatDate, TelechatAgendaItem
-
+from ietf.review.utils import review_requests_to_list_for_docs
 
 def get_agenda_date(date=None):
     if not date:
@@ -152,6 +152,8 @@ def fill_in_agenda_docs(date, sections, matches=None):
         matches = Document.objects.filter(docevent__telechatdocevent__telechat_date=date)
         matches = matches.select_related("stream", "group").distinct()
 
+    review_requests_for_docs = review_requests_to_list_for_docs(matches)
+
     for doc in matches:
         if doc.latest_event(TelechatDocEvent, type="scheduled_for_telechat").telechat_date != date:
             continue
@@ -174,6 +176,8 @@ def fill_in_agenda_docs(date, sections, matches=None):
                 e = doc.latest_event(ConsensusDocEvent, type="changed_consensus")
                 if e and (e.consensus != None):
                     doc.consensus = "Yes" if e.consensus else "No"
+
+            doc.review_requests = review_requests_for_docs.get(doc.pk, [])
         elif doc.type_id == "conflrev":
             doc.conflictdoc = doc.relateddocument_set.get(relationship__slug='conflrev').target.document
         elif doc.type_id == "charter":

ietf/review/import_from_review_tool.py

@@ -16,8 +16,8 @@
 import datetime, re, itertools
 from collections import namedtuple
 from django.db import connections
-from ietf.review.models import (ReviewRequest, ReviewerSettings, ReviewResultName,
-                                ReviewRequestStateName, ReviewTypeName, ReviewTeamResult,
+from ietf.review.models import (ReviewRequest, ReviewerSettings, ReviewResultName, ResultUsedInReviewTeam,
+                                ReviewRequestStateName, ReviewTypeName, TypeUsedInReviewTeam,
                                 UnavailablePeriod, NextReviewerInTeam)
 from ietf.group.models import Group, Role, RoleName
 from ietf.person.models import Person, Email, Alias
@@ -129,6 +129,10 @@ def parse_timestamp(t):
                     today = datetime.date.today()
                     end_date = unavailable_until.date()
                     if end_date >= today:
+                        if end_date >= datetime.date(2020, 1, 1):
+                            # convert hacked end dates to indefinite
+                            end_date = None
+
                         UnavailablePeriod.objects.filter(person=email.person, team=team).delete()
 
                         UnavailablePeriod.objects.create(
@@ -162,7 +166,10 @@ def parse_timestamp(t):
             summaries = [v.strip().lower() for v in row.value.split(";") if v.strip()]
 
             for s in summaries:
-                ReviewTeamResult.objects.get_or_create(team=team, result=results[s])
+                ResultUsedInReviewTeam.objects.get_or_create(team=team, result=results[s])
+
+for t in ReviewTypeName.objects.filter(slug__in=["early", "lc", "telechat"]):
+    TypeUsedInReviewTeam.objects.get_or_create(team=team, type=t)
 
 # review requests
 
@@ -186,8 +193,8 @@ def parse_timestamp(t):
 
 requested_re = re.compile("(?:ADDED docname =>|Created: remote=|AUTO UPDATED status TO new|CHANGED status FROM [^ ]+ => new|CHANGE status done => working)")
 
-add_docstatus_re = re.compile('([a-zA-Z-_]+) ADD docstatus => (\w+)')
-update_docstatus_re = re.compile('([a-zA-Z-_]+) (?:UPDATE|CHANGE) docstatus \w+ => (\w+)')
+add_docstatus_re = re.compile('([a-zA-Z-_@.]+) ADD docstatus => (\w+)')
+update_docstatus_re = re.compile('([a-zA-Z-_@.]+) (?:UPDATE|CHANGE) docstatus \w+ => (\w+)')
 iesgstatus_re = re.compile('(?:ADD|ADDED|CHANGED) iesgstatus (?:FROM )?(?:[^ ]+ )?=> ([a-zA-Z-_]+)?')
 
 deadline_re = re.compile('(?:ADD|ADDED|CHANGED) deadline (?:FROM )?(?:[^ ]+ )?=> ([1-9][0-9]+)')
@@ -196,6 +203,8 @@ def parse_timestamp(t):
 
 close_states = ["done", "rejected", "withdrawn", "noresponse"]
 
+document_blacklist = set([(u"tsvdir", u"draft-arkko-ipv6-transition-guidelines-09 ")])
+
 with db_con.cursor() as c:
     c.execute("""select docname, time, who, text from doclog where
                         text like 'Created: remote=%'
@@ -214,6 +223,9 @@ def parse_timestamp(t):
                      or text like '%CHANGED iesgstatus % => %'
                    order by docname, time asc;""")
     for docname, rows in itertools.groupby(namedtuplefetchall(c), lambda row: row.docname):
+        if (team.acronym, docname) in document_blacklist:
+            continue # ignore
+
         branches = {}
 
         latest_requested = None
@@ -233,12 +245,12 @@ def parse_timestamp(t):
             else:
                 if "ADD docstatus" in row.text:
                     m = add_docstatus_re.match(row.text)
-                    assert m, 'row.text "{}" does not match add regexp'.format(row.text)
+                    assert m, 'row.text "{}" does not match add regexp {}'.format(row.text, docname)
                     membername, state = m.groups()
                     used = True
                 elif "UPDATE docstatus" in row.text or "CHANGE docstatus" in row.text:
                     m = update_docstatus_re.match(row.text)
-                    assert m, 'row.text "{}" does not match update regexp'.format(row.text)
+                    assert m, 'row.text "{}" does not match update regexp {}'.format(row.text, docname)
                     membername, state = m.groups()
                     used = True
 
@@ -332,6 +344,9 @@ def parse_timestamp(t):
     c.execute("select * from reviews order by reviewid;")
 
     for row in namedtuplefetchall(c):
+        if (team.acronym, row.docname) in document_blacklist:
+            continue # ignore
+
         meta = doc_metadata.get((row.docname, row.version))
         if not meta:
             meta = doc_metadata.get(row.docname)
@@ -345,7 +360,7 @@ def parse_timestamp(t):
         if row.summary == "noresponse":
             reviewed_rev = ""
 
-        event_collection = None
+        event_collection = {}
         branches = document_history.get(row.docname)
         if not branches:
             print "WARNING: no history for", row.docname

ietf/review/migrations/0001_initial.py

@@ -63,7 +63,7 @@ class Migration(migrations.Migration):
             bases=(models.Model,),
         ),
         migrations.CreateModel(
-            name='ReviewTeamResult',
+            name='ResultUsedInReviewTeam',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('result', models.ForeignKey(to='name.ReviewResultName')),
@@ -73,6 +73,17 @@ class Migration(migrations.Migration):
             },
             bases=(models.Model,),
         ),
+        migrations.CreateModel(
+            name='TypeUsedInReviewTeam',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('team', models.ForeignKey(to='group.Group')),
+                ('type', models.ForeignKey(to='name.ReviewTypeName')),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
         migrations.CreateModel(
             name='ReviewWish',
             fields=[

ietf/review/models.py

@@ -66,7 +66,7 @@ class ReviewWish(models.Model):
     def __unicode__(self):
         return u"{} wishes to review {} in {}".format(self.person, self.doc.name, self.team.acronym)
 
-class ReviewTeamResult(models.Model):
+class ResultUsedInReviewTeam(models.Model):
     """Captures that a result name is valid for a given team for new
     reviews. This also implicitly defines which teams are review
     teams - if there are no possible review results valid for a given
@@ -75,7 +75,16 @@ class ReviewTeamResult(models.Model):
     result      = models.ForeignKey(ReviewResultName)
 
     def __unicode__(self):
-        return u"{} in {}".format(self.result.name, self.group.acronym)
+        return u"{} in {}".format(self.result.name, self.team.acronym)
+
+class TypeUsedInReviewTeam(models.Model):
+    """Captures that a type name is valid for a given team for new
+    reviews. """
+    team        = models.ForeignKey(Group)
+    type        = models.ForeignKey(ReviewTypeName)
+
+    def __unicode__(self):
+        return u"{} in {}".format(self.type.name, self.team.acronym)
 
 class NextReviewerInTeam(models.Model):
     team        = models.ForeignKey(Group)
@@ -97,7 +106,7 @@ class ReviewRequest(models.Model):
     time          = models.DateTimeField(default=datetime.datetime.now)
     type          = models.ForeignKey(ReviewTypeName)
     doc           = models.ForeignKey(Document, related_name='reviewrequest_set')
-    team          = models.ForeignKey(Group, limit_choices_to=~models.Q(reviewteamresult=None))
+    team          = models.ForeignKey(Group, limit_choices_to=~models.Q(resultusedinreviewteam=None))
     deadline      = models.DateField()
     requested_by  = models.ForeignKey(Person)
     requested_rev = models.CharField(verbose_name="requested revision", max_length=16, blank=True, help_text="Fill in if a specific revision is to be reviewed, e.g. 02")