Integrate community lists for groups with the existing group documents

page. Each WG/RG now gets a list with an initial set of rules to
populate the list.

Refine the community list management interface a bit to support the
group lists better - group lists aren't connected to the usual track
icons so need to be able to add/remove individual drafts.

Change the "name contains" rule to support regular expressions to
enable each group to have a default replacement for the previously
implemented "related documents" search. Maintain a materialized view
of the regexp-matched drafts with a call in the submit code to avoid
having to scan all drafts/~1000 group rules all the time.
 - Legacy-Id: 10963

Author: OleLaursen

ietf/community/forms.py

@@ -87,6 +87,9 @@ def restrict_state(state_type, slug=None):
         for name, f in self.fields.iteritems():
             f.required = True
 
+    def clean_text(self):
+        return self.cleaned_data["text"].strip().lower() # names are always lower case
+
 
 class SubscriptionForm(forms.ModelForm):
     def __init__(self, user, clist, *args, **kwargs):

ietf/community/migrations/0003_cleanup.py

@@ -88,7 +88,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='searchrule',
             name='text',
-            field=models.CharField(default=b'', max_length=255, blank=True),
+            field=models.CharField(default=b'', max_length=255, verbose_name=b'Text/RegExp', blank=True),
             preserve_default=True,
         ),
         migrations.RemoveField(
@@ -98,7 +98,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='searchrule',
             name='rule_type',
-            field=models.CharField(max_length=30, choices=[(b'group', b'All I-Ds associated with a particular group'), (b'area', b'All I-Ds associated with all groups in a particular Area'), (b'group_rfc', b'All RFCs associated with a particular group'), (b'area_rfc', b'All RFCs associated with all groups in a particular Area'), (b'state_iab', b'All I-Ds that are in a particular IAB state'), (b'state_iana', b'All I-Ds that are in a particular IANA state'), (b'state_iesg', b'All I-Ds that are in a particular IESG state'), (b'state_irtf', b'All I-Ds that are in a particular IRTF state'), (b'state_ise', b'All I-Ds that are in a particular ISE state'), (b'state_rfceditor', b'All I-Ds that are in a particular RFC Editor state'), (b'state_ietf', b'All I-Ds that are in a particular Working Group state'), (b'author', b'All I-Ds with a particular author'), (b'author_rfc', b'All RFCs with a particular author'), (b'ad', b'All I-Ds with a particular responsible AD'), (b'shepherd', b'All I-Ds with a particular document shepherd'), (b'name_contains', b'All I-Ds with particular text in the name')]),
+            field=models.CharField(max_length=30, choices=[(b'group', b'All I-Ds associated with a particular group'), (b'area', b'All I-Ds associated with all groups in a particular Area'), (b'group_rfc', b'All RFCs associated with a particular group'), (b'area_rfc', b'All RFCs associated with all groups in a particular Area'), (b'state_iab', b'All I-Ds that are in a particular IAB state'), (b'state_iana', b'All I-Ds that are in a particular IANA state'), (b'state_iesg', b'All I-Ds that are in a particular IESG state'), (b'state_irtf', b'All I-Ds that are in a particular IRTF state'), (b'state_ise', b'All I-Ds that are in a particular ISE state'), (b'state_rfceditor', b'All I-Ds that are in a particular RFC Editor state'), (b'state_ietf', b'All I-Ds that are in a particular Working Group state'), (b'author', b'All I-Ds with a particular author'), (b'author_rfc', b'All RFCs with a particular author'), (b'ad', b'All I-Ds with a particular responsible AD'), (b'shepherd', b'All I-Ds with a particular document shepherd'), (b'name_contains', b'All I-Ds with particular text/regular expression in the name')]),
             preserve_default=True,
         ),
         migrations.AlterUniqueTogether(
@@ -111,4 +111,10 @@ class Migration(migrations.Migration):
             field=models.CharField(default=b'all', max_length=30, choices=[(b'all', b'All changes'), (b'significant', b'Only significant state changes')]),
             preserve_default=True,
         ),
+        migrations.AddField(
+            model_name='searchrule',
+            name='name_contains_index',
+            field=models.ManyToManyField(to='doc.Document'),
+            preserve_default=True,
+        ),
     ]

ietf/community/migrations/0004_cleanup_data.py

@@ -194,6 +194,31 @@ def fill_in_notify_on(apps, schema_editor):
     EmailSubscription.objects.filter(significant=False, notify_on="all")
     EmailSubscription.objects.filter(significant=True, notify_on="significant")
 
+def add_group_community_lists(apps, schema_editor):
+    Group = apps.get_model("group", "Group")
+    Document = apps.get_model("doc", "Document")
+    State = apps.get_model("doc", "State")
+    CommunityList = apps.get_model("community", "CommunityList")
+    SearchRule = apps.get_model("community", "SearchRule")
+
+    active_state = State.objects.get(slug="active", type="draft")
+    rfc_state = State.objects.get(slug="rfc", type="draft")
+
+    for g in Group.objects.filter(type__in=("rg", "wg")):
+        clist = CommunityList.objects.filter(group=g).first()
+        if clist:
+            SearchRule.objects.get_or_create(community_list=clist, rule_type="group", group=g, state=active_state)
+            SearchRule.objects.get_or_create(community_list=clist, rule_type="group_rfc", group=g, state=rfc_state)
+            r, _ = SearchRule.objects.get_or_create(community_list=clist, rule_type="name_contains", text=r"^draft-[^-]+-%s-" % g.acronym, state=active_state)
+            r.name_contains_index = Document.objects.filter(docalias__name__regex=r.text)
+
+        else:
+            clist = CommunityList.objects.create(group=g)
+            SearchRule.objects.create(community_list=clist, rule_type="group", group=g, state=active_state)
+            SearchRule.objects.create(community_list=clist, rule_type="group_rfc", group=g, state=rfc_state)
+            r = SearchRule.objects.create(community_list=clist, rule_type="name_contains", text=r"^draft-[^-]+-%s-" % g.acronym, state=active_state)
+            r.name_contains_index = Document.objects.filter(docalias__name__regex=r.text)
+
 def noop(apps, schema_editor):
     pass
 
@@ -209,6 +234,7 @@ class Migration(migrations.Migration):
         migrations.RunPython(move_email_subscriptions_to_preregistered_email, noop),
         migrations.RunPython(get_rid_of_empty_lists, noop),
         migrations.RunPython(fill_in_notify_on, noop),
+        migrations.RunPython(add_group_community_lists, noop),
         migrations.RemoveField(
             model_name='searchrule',
             name='value',

ietf/community/models.py

@@ -1,6 +1,7 @@
 from django.contrib.auth.models import User
 from django.db import models
 from django.db.models import signals
+from django.core.urlresolvers import reverse as urlreverse
 
 from ietf.doc.models import Document, DocEvent, State
 from ietf.group.models import Group
@@ -22,6 +23,13 @@ def long_name(self):
     def __unicode__(self):
         return self.long_name()
 
+    def get_absolute_url(self):
+        if self.user:
+            return urlreverse("community_personal_view_list", kwargs={ 'username': self.user.username })
+        elif self.group:
+            return urlreverse("group_docs", kwargs={ 'acronym': self.group.acronym })
+        return ""
+
 
 class SearchRule(models.Model):
     # these types define the UI for setting up the rule, and also
@@ -47,7 +55,7 @@ class SearchRule(models.Model):
 
         ('shepherd', 'All I-Ds with a particular document shepherd'),
 
-        ('name_contains', 'All I-Ds with particular text in the name'),
+        ('name_contains', 'All I-Ds with particular text/regular expression in the name'),
     ]
 
     community_list = models.ForeignKey(CommunityList)
@@ -57,7 +65,13 @@ class SearchRule(models.Model):
     state = models.ForeignKey(State, blank=True, null=True)
     group = models.ForeignKey(Group, blank=True, null=True)
     person = models.ForeignKey(Person, blank=True, null=True)
-    text = models.CharField(max_length=255, blank=True, default="")
+    text = models.CharField(verbose_name="Text/RegExp", max_length=255, blank=True, default="")
+
+    # store a materialized view/index over which documents are matched
+    # by the name_contains rule to avoid having to scan the whole
+    # database - we update this manually when the rule is changed and
+    # when new documents are submitted
+    name_contains_index = models.ManyToManyField(Document)
 
 
 class EmailSubscription(models.Model):

ietf/community/tests.py

@@ -7,6 +7,8 @@
 
 from ietf.community.models import CommunityList, SearchRule, EmailSubscription
 from ietf.community.utils import docs_matching_community_list_rule, community_list_rules_matching_doc
+from ietf.community.utils import reset_name_contains_index_for_rule
+from ietf.group.utils import setup_default_community_list_for_group
 from ietf.doc.models import State
 from ietf.doc.utils import add_state_change_event
 from ietf.person.models import Person, Email
@@ -34,7 +36,8 @@ def test_rule_matching(self):
 
         rule_shepherd = SearchRule.objects.create(rule_type="shepherd", state=State.objects.get(type="draft", slug="active"), person=draft.shepherd.person, community_list=clist)
 
-        rule_name_contains = SearchRule.objects.create(rule_type="name_contains", state=State.objects.get(type="draft", slug="active"), text="-".join(draft.name.split("-")[2:]), community_list=clist)
+        rule_name_contains = SearchRule.objects.create(rule_type="name_contains", state=State.objects.get(type="draft", slug="active"), text="draft-.*" + "-".join(draft.name.split("-")[2:]), community_list=clist)
+        reset_name_contains_index_for_rule(rule_name_contains)
 
         # doc -> rules
         matching_rules = list(community_list_rules_matching_doc(draft))
@@ -79,7 +82,7 @@ def test_view_list(self):
         self.assertEqual(r.status_code, 200)
         self.assertTrue(draft.name in r.content)
 
-    def test_manage_list(self):
+    def test_manage_personal_list(self):
         draft = make_test_data()
 
         url = urlreverse("community_personal_manage_list", kwargs={ "username": "plain" })
@@ -94,6 +97,17 @@ def test_manage_list(self):
         clist = CommunityList.objects.get(user__username="plain")
         self.assertTrue(clist.added_docs.filter(pk=draft.pk))
 
+        # document shows up on GET
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertTrue(draft.name in r.content)
+
+        # remove document
+        r = self.client.post(url, { "action": "remove_document", "document": draft.pk })
+        self.assertEqual(r.status_code, 302)
+        clist = CommunityList.objects.get(user__username="plain")
+        self.assertTrue(not clist.added_docs.filter(pk=draft.pk))
+        
         # add rule
         r = self.client.post(url, {
             "action": "add_rule",
@@ -105,6 +119,17 @@ def test_manage_list(self):
         clist = CommunityList.objects.get(user__username="plain")
         self.assertTrue(clist.searchrule_set.filter(rule_type="author_rfc"))
 
+        # add name_contains rule
+        r = self.client.post(url, {
+            "action": "add_rule",
+            "rule_type": "name_contains",
+            "name_contains-text": "draft.*mars",
+            "name_contains-state": State.objects.get(type="draft", slug="active").pk,
+        })
+        self.assertEqual(r.status_code, 302)
+        clist = CommunityList.objects.get(user__username="plain")
+        self.assertTrue(clist.searchrule_set.filter(rule_type="name_contains"))
+
         # rule shows up on GET
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
@@ -121,50 +146,61 @@ def test_manage_list(self):
         clist = CommunityList.objects.get(user__username="plain")
         self.assertTrue(not clist.searchrule_set.filter(rule_type="author_rfc"))
 
-    def test_track_untrack_document_for_personal_list_through_ajax(self):
+    def test_manage_group_list(self):
+        draft = make_test_data()
+
+        url = urlreverse("community_group_manage_list", kwargs={ "acronym": draft.group.acronym })
+        setup_default_community_list_for_group(draft.group)
+        login_testing_unauthorized(self, "marschairman", url)
+
+        # test GET, rest is tested with personal list
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+
+    def test_track_untrack_document(self):
         draft = make_test_data()
 
         url = urlreverse("community_personal_track_document", kwargs={ "username": "plain", "name": draft.name })
         login_testing_unauthorized(self, "plain", url)
 
         # track
-        r = self.client.post(url, HTTP_X_REQUESTED_WITH='XMLHttpRequest')
+        r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
-        self.assertEqual(json.loads(r.content)["success"], True)
+
+        r = self.client.post(url)
+        self.assertEqual(r.status_code, 302)
         clist = CommunityList.objects.get(user__username="plain")
         self.assertEqual(list(clist.added_docs.all()), [draft])
 
         # untrack
         url = urlreverse("community_personal_untrack_document", kwargs={ "username": "plain", "name": draft.name })
-        r = self.client.post(url, HTTP_X_REQUESTED_WITH='XMLHttpRequest')
+        r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
-        self.assertEqual(json.loads(r.content)["success"], True)
+
+        r = self.client.post(url)
+        self.assertEqual(r.status_code, 302)
         clist = CommunityList.objects.get(user__username="plain")
         self.assertEqual(list(clist.added_docs.all()), [])
 
-    def test_track_untrack_document_for_group_list(self):
+    def test_track_untrack_document_through_ajax(self):
         draft = make_test_data()
 
-        url = urlreverse("community_group_track_document", kwargs={ "acronym": draft.group.acronym, "name": draft.name })
-        login_testing_unauthorized(self, "marschairman", url)
+        url = urlreverse("community_personal_track_document", kwargs={ "username": "plain", "name": draft.name })
+        login_testing_unauthorized(self, "plain", url)
 
         # track
-        r = self.client.get(url)
+        r = self.client.post(url, HTTP_X_REQUESTED_WITH='XMLHttpRequest')
         self.assertEqual(r.status_code, 200)
-
-        r = self.client.post(url)
-        self.assertEqual(r.status_code, 302)
-        clist = CommunityList.objects.get(group__acronym=draft.group.acronym)
+        self.assertEqual(json.loads(r.content)["success"], True)
+        clist = CommunityList.objects.get(user__username="plain")
         self.assertEqual(list(clist.added_docs.all()), [draft])
 
         # untrack
-        url = urlreverse("community_group_untrack_document", kwargs={ "acronym": draft.group.acronym, "name": draft.name })
-        r = self.client.get(url)
+        url = urlreverse("community_personal_untrack_document", kwargs={ "username": "plain", "name": draft.name })
+        r = self.client.post(url, HTTP_X_REQUESTED_WITH='XMLHttpRequest')
         self.assertEqual(r.status_code, 200)
-
-        r = self.client.post(url)
-        self.assertEqual(r.status_code, 302)
-        clist = CommunityList.objects.get(group__acronym=draft.group.acronym)
+        self.assertEqual(json.loads(r.content)["success"], True)
+        clist = CommunityList.objects.get(user__username="plain")
         self.assertEqual(list(clist.added_docs.all()), [])
 
     def test_csv(self):
@@ -190,6 +226,17 @@ def test_csv(self):
         # this is a simple-minded test, we don't actually check the fields
         self.assertTrue(draft.name in r.content)
 
+    def test_csv_for_group(self):
+        draft = make_test_data()
+
+        url = urlreverse("community_group_csv", kwargs={ "acronym": draft.group.acronym })
+
+        setup_default_community_list_for_group(draft.group)
+
+        # test GET, rest is tested with personal list
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+
     def test_feed(self):
         draft = make_test_data()
 
@@ -217,6 +264,17 @@ def test_feed(self):
         self.assertEqual(r.status_code, 200)
         self.assertTrue('<entry>' not in r.content)
 
+    def test_feed_for_group(self):
+        draft = make_test_data()
+
+        url = urlreverse("community_group_feed", kwargs={ "acronym": draft.group.acronym })
+
+        setup_default_community_list_for_group(draft.group)
+
+        # test GET, rest is tested with personal list
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        
     def test_subscription(self):
         draft = make_test_data()
 
@@ -254,6 +312,19 @@ def test_subscription(self):
         self.assertEqual(r.status_code, 302)
         self.assertEqual(EmailSubscription.objects.filter(community_list=clist, email=email, notify_on="significant").count(), 0)
 
+    def test_subscription_for_group(self):
+        draft = make_test_data()
+
+        url = urlreverse("community_group_subscription", kwargs={ "acronym": draft.group.acronym })
+
+        setup_default_community_list_for_group(draft.group)
+
+        login_testing_unauthorized(self, "marschairman", url)
+
+        # test GET, rest is tested with personal list
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        
     def test_notification(self):
         draft = make_test_data()
 

ietf/community/urls.py

@@ -1,20 +1,13 @@
 from django.conf.urls import patterns, url
 
 
-urlpatterns = patterns('ietf.community.views',
-    url(r'^personal/(?P<username>[^/]+)/$', 'view_list', name='community_personal_view_list'),
-    url(r'^personal/(?P<username>[^/]+)/manage/$', 'manage_list', name='community_personal_manage_list'),
-    url(r'^personal/(?P<username>[^/]+)/trackdocument/(?P<name>[^/]+)/$', 'track_document', name='community_personal_track_document'),
-    url(r'^personal/(?P<username>[^/]+)/untrackdocument/(?P<name>[^/]+)/$', 'untrack_document', name='community_personal_untrack_document'),
-    url(r'^personal/(?P<username>[^/]+)/csv/$', 'export_to_csv', name='community_personal_csv'),
-    url(r'^personal/(?P<username>[^/]+)/feed/$', 'feed', name='community_personal_feed'),
-    url(r'^personal/(?P<username>[^/]+)/subscription/$', 'subscription', name='community_personal_subscription'),
+urlpatterns = patterns('',
+    url(r'^personal/(?P<username>[^/]+)/$', 'ietf.community.views.view_list', name='community_personal_view_list'),
+    url(r'^personal/(?P<username>[^/]+)/manage/$', 'ietf.community.views.manage_list', name='community_personal_manage_list'),
+    url(r'^personal/(?P<username>[^/]+)/trackdocument/(?P<name>[^/]+)/$', 'ietf.community.views.track_document', name='community_personal_track_document'),
+    url(r'^personal/(?P<username>[^/]+)/untrackdocument/(?P<name>[^/]+)/$', 'ietf.community.views.untrack_document', name='community_personal_untrack_document'),
+    url(r'^personal/(?P<username>[^/]+)/csv/$', 'ietf.community.views.export_to_csv', name='community_personal_csv'),
+    url(r'^personal/(?P<username>[^/]+)/feed/$', 'ietf.community.views.feed', name='community_personal_feed'),
+    url(r'^personal/(?P<username>[^/]+)/subscription/$', 'ietf.community.views.subscription', name='community_personal_subscription'),
 
-    url(r'^group/(?P<acronym>[\w.@+-]+)/$', 'view_list', name='community_group_view_list'),
-    url(r'^group/(?P<acronym>[\w.@+-]+)/manage/$', 'manage_list', name='community_group_manage_list'),
-    url(r'^group/(?P<acronym>[\w.@+-]+)/trackdocument/(?P<name>[^/]+)/$', 'track_document', name='community_group_track_document'),
-    url(r'^group/(?P<acronym>[\w.@+-]+)/untrackdocument/(?P<name>[^/]+)/$', 'untrack_document', name='community_group_untrack_document'),
-    url(r'^group/(?P<acronym>[\w.@+-]+)/csv/$', 'export_to_csv', name='community_group_csv'),
-    url(r'^group/(?P<acronym>[\w.@+-]+)/feed/$', 'feed', name='community_group_feed'),
-    url(r'^group/(?P<acronym>[\w.@+-]+)/subscription/$', 'subscription', name='community_group_subscription'),
 )