Merged in an update from trunk@9942.

 - Legacy-Id: 9961

Author: levkowetz

bin/test-crawl

@@ -2,7 +2,6 @@
 
 import os, sys, re, datetime, argparse, traceback, tempfile, json, subprocess
 import html5lib
-import debug    # pyflakes:ignore
 import random
 
 # Set up import path to find our own Django
@@ -33,6 +32,8 @@ parser.add_argument('--random', action='store_true',
 parser.add_argument('--validate-all', dest='validate_all', action='store_true', default=False,
                     help='Run html 5 validation on all pages, without skipping similar urls. '
                     '(The default is to only run validation on one of /foo/1/, /foo/2/, /foo/3/, etc.)')
+parser.add_argument('-v', '--verbose', action='store_true', default=False,
+                    help='Be more verbose')
 
 args = parser.parse_args()
 
@@ -44,6 +45,9 @@ import django.test
 
 django.setup()
 
+# This needs to come after we set up sys path to include the local django
+import debug    # pyflakes:ignore
+
 # prevent memory from leaking when settings.DEBUG=True
 from django.db import connection
 class DontSaveQueries(object):
@@ -59,6 +63,11 @@ MAX_URL_LENGTH = 500
 
 # --- Functions ---
 
+def note(s):
+    if args.verbose:
+        sys.stderr.write(s)
+        sys.stderr.write('\n')
+
 def strip_url(url):
     if url.startswith("http://testserver"):
         url = url[len("http://testserver"):]
@@ -105,26 +114,45 @@ def extract_tastypie_urls(content):
 
 def check_html_valid(url, response, args):
     global parser, validated_urls, doc_types, warnings
-    # These URLs have known issues, skip them until those are fixed
-    if re.search('(/secr|admin/)|/doc/.*/edit/info/', url):
-        log("%s blacklisted; skipping HTML validation" % url)
-        return
     key = url
     if not args.validate_all:
         # derive a key for urls like this by replacing primary keys
-        key = re.sub("/[0-9.]+/",   "/nnnn/", key)
-        key = re.sub("/.+@.+/",    "/x@x.org/", key)
-        key = re.sub("#.*$",       "", key)
         key = re.sub("\?.*$",       "", key)
+        key = re.sub("#.*$",       "", key)
+        key = re.sub("/.+@.+/",    "/x@x.org/", key)
+        key = re.sub("/[0-9.]+/",   "/nnnn/", key)
+        key = re.sub("/[0-9.]+/",   "/mmmm/", key)
+        key = re.sub("/ag/[a-z0-9-]+/",   "/ag/foo/", key)
+        key = re.sub("/area/[a-z0-9-]+/",   "/area/foo/", key)
+        key = re.sub("/bcp[0-9]+/",   "/bcpnnn/", key)
+        key = re.sub("/conflict-review-[a-z0-9-]+/",   "/conflrev-foo/", key)
+        key = re.sub("/dir/[a-z0-9-]+/",   "/dir/foo/", key)
+        key = re.sub("/draft-[a-z0-9-]+/",   "/draft-foo/", key)
+        key = re.sub("/group/[a-z0-9-]+/",   "/group/foo/", key)
+        key = re.sub("/ipr/search/.*",   "/ipr/search/", key)
+        key = re.sub("/release/[0-9dev.]+/",   "/release/n.n.n/", key)
         key = re.sub("/rfc[0-9]+/",   "/rfcnnnn/", key)
-        key = re.sub("/wg/[a-z0-9-]+/",   "/wg/foo/", key)
         key = re.sub("/rg/[a-z0-9-]+/",   "/rg/foo/", key)
-        key = re.sub("/ipr/[0-9]+/",   "/ipr/nnnn/", key)
-        key = re.sub("/draft-[a-z0-9-]+/",   "/draft-foo/", key)
+        key = re.sub("/secr/srec/nnnn/[0-9a-z-]+/", "/secr/sreq/nn/bar/", key)
+        key = re.sub("/state/[a-z0-9-]+/",   "/state/foo/", key)
+        key = re.sub("/state/[a-z0-9-]+/[a-z0-9-]+/",   "/state/foo/bar/", key)
+        key = re.sub("/status-change-[a-z0-9-]+/",   "/statchg-foo/", key)
+        key = re.sub("/std[0-9]+/",   "/stdnnn/", key)
+        key = re.sub("/submit/status/nnnn/[0-9a-f]+/", "/submit/status/nnnn/bar/", key)
+        key = re.sub("/team/[a-z0-9-]+/",   "/team/foo/", key)
+        key = re.sub("/wg/[a-z0-9-]+/",   "/wg/foo/", key)
+
         for slug in doc_types:
             key = re.sub("/%s-.*/"%slug, "/%s-nnnn/"%slug, key)
 
     if not key in validated_urls:
+        note('Validate: %-32s: %s' % (url[:32], key))
+        # These URLs have known issues, skip them until those are fixed
+        if re.search('(/secr|admin/)|/doc/.*/edit/info/', url):
+            log("%s blacklisted; skipping HTML validation" % url)
+            validated_urls[key] = True
+            return
+
         if hasattr(response, "content"):
             content = response.content
         else:
@@ -156,6 +184,14 @@ def check_html_valid(url, response, args):
                                 (pos, code))
                 warnings += 1
 
+def skip_url(url):
+    for pattern in (
+            "^/community/[0-9]+/remove_document/",
+            "^/community/personal/",
+        ):
+        if re.search(pattern, url):
+            return True
+    return False
 
 def log(s):
     print(s)
@@ -243,6 +279,9 @@ if __name__ == "__main__":
 
         visited.add(url)
 
+        if skip_url(url):
+            continue
+
         try:
             timestamp = datetime.datetime.now()
             r = client.get(url, secure=True, follow=True)
@@ -298,7 +337,7 @@ if __name__ == "__main__":
                         log("=============")
 
             else:
-                tags.append(u"FAIL for %s\n   (from %s)" % (url, referrer))
+                tags.append(u"FAIL  (from %s)" % (referrer, ))
                 errors += 1
 
             if elapsed.total_seconds() > slow_threshold:

changelog

@@ -1,3 +1,40 @@
+ietfdb (6.3.0) ietf; urgency=medium
+
+  **Active area, directorate, team, and area group pages**
+
+  This release provides new group overview pages for active areas, area
+  groups, teams, and directorates.  It also improves the error reporting when
+  drafts are submitted with invalid XML, providing line numbers and specifics
+  for the issues found.  Additionally, there are a few bug fixes.
+
+  * Merged in [9924] from rjsparks@nostrum.com:
+    Added views of active areas, area groups, teams, and directorates.  Added
+    navigation to those views to the base menus.  Unified the URL patterns
+    shared between group/urls and group/urls_info, exposing the same view at,
+    e.g., /group/stir and /wg/stir/.  Improved testing, primarily of
+    group/info.py 
+
+  * Merged in [9901] from rcross@amsl.com:
+    Fixed matching audio file names with rooms.   
+
+  * Provided a document's rfc number (if any) as part of the document fields 
+    exposed in the json api.
+
+  * Changed the code for meta-data extraction from xml files to not break 
+    if a sought-after element is missing.
+
+  * Updated the meta-data error message to say 'errors ... below' instead of
+    'errors ... above', to match the relative placement of text and error
+    indications.
+
+  * Improved the error reporting for invalid xml file submissions.
+
+  * Improved the handling of failed xml2rfc conversions when no txt document
+    is submitted.
+
+ -- Henrik Levkowetz <henrik@levkowetz.com>  01 Aug 2015 14:52:37 +0000
+
+
 ietfdb (6.2.0) ietf; urgency=medium
 
   **XML-Only Draft Submission**

ietf/__init__.py

@@ -2,13 +2,13 @@
 import checks                           # pyflakes:ignore
 
 # Don't add patch number here:
-__version__ = "6.2.1.dev0"
+__version__ = "6.3.1.dev0"
 
 # set this to ".p1", ".p2", etc. after patching
 __patch__   = ""                        
 
 __date__    = "$Date$"
 
-__rev__     = "$Rev$ (dev) Latest release: Rev. 9880 "
+__rev__     = "$Rev$ (dev) Latest release: Rev. 9940 "
 
 __id__      = "$Id$"

ietf/doc/mails.py

@@ -179,7 +179,7 @@ def generate_approval_mail_approved(request, doc):
         doc.group.name_with_wg = doc.group.name + " Working Group"
         if doc.group.list_email:
             cc.append("%s mailing list <%s>" % (doc.group.acronym, doc.group.list_email))
-        cc.append("%s chair <%s-chairs@tools.ietf.org>" % (doc.group.acronym, doc.group.acronym))
+        cc.append("%s chair <%s-chairs@ietf.org>" % (doc.group.acronym, doc.group.acronym))
     else:
         doc.group.name_with_wg = doc.group.name
 

ietf/doc/resources.py

@@ -1,6 +1,6 @@
 # Autogenerated by the mkresources management command 2014-12-14 19:50
 from tastypie.resources import ModelResource
-from tastypie.fields import ToOneField, ToManyField
+from tastypie.fields import ToOneField, ToManyField, CharField
 from tastypie.constants import ALL, ALL_WITH_RELATIONS
 
 from ietf import api
@@ -86,6 +86,7 @@ class DocumentResource(ModelResource):
     states           = ToManyField(StateResource, 'states', null=True)
     tags             = ToManyField(DocTagNameResource, 'tags', null=True)
     authors          = ToManyField(EmailResource, 'authors', null=True)
+    rfc              = CharField(attribute='rfc_number', null=True)
     class Meta:
         queryset = Document.objects.all()
         #resource_name = 'document'

ietf/doc/templatetags/active_groups_menu.py

@@ -0,0 +1,14 @@
+from django import template
+from django.template.loader import render_to_string
+
+from ietf.name.models import GroupTypeName
+
+register = template.Library()
+
+@register.simple_tag
+def active_groups_menu():
+    parents = GroupTypeName.objects.filter(slug__in=['ag','area','team','dir'])
+    for p in parents:
+        p.menu_url = '/%s/'%p.slug
+    return render_to_string('base/menu_active_groups.html', { 'parents': parents })
+

ietf/doc/utils.py

@@ -467,7 +467,7 @@ def collect_email_addresses(emails, doc):
             if role.email.address not in emails:
                 emails[role.email.address] = '"%s"' % (role.person.name)
         if doc.group.type.slug == 'wg':
-            address = '%s-ads@tools.ietf.org' % doc.group.acronym
+            address = '%s-ads@ietf.org' % doc.group.acronym
             if address not in emails:
                 emails[address] = '"%s-ads"' % (doc.group.acronym)
         elif doc.group.type.slug == 'rg':

ietf/doc/views_draft.py

@@ -13,6 +13,8 @@
 from django.template.defaultfilters import pluralize
 from django.contrib import messages
 
+import debug                            # pyflakes:ignore
+
 from ietf.doc.models import ( Document, DocAlias, RelatedDocument, State,
     StateType, DocEvent, ConsensusDocEvent, TelechatDocEvent, WriteupDocEvent, IESG_SUBSTATE_TAGS,
     save_document_in_history )
@@ -582,7 +584,7 @@ def to_iesg(request,name):
             doc.save()
 
             extra = {}
-            extra['Cc'] = "%s-chairs@tools.ietf.org, iesg-secretary@ietf.org, %s" % (doc.group.acronym,doc.notify)
+            extra['Cc'] = "%s-chairs@ietf.org, iesg-secretary@ietf.org, %s" % (doc.group.acronym,doc.notify)
             send_mail(request=request,
                       to = doc.ad.email_address(),
                       frm = login.formatted_email(),

ietf/group/edit.py

@@ -167,14 +167,29 @@ def get_or_create_initial_charter(group, group_type):
     return charter
 
 @login_required
-def submit_initial_charter(request, group_type, acronym=None):
-    if not can_manage_group_type(request.user, group_type):
-        return HttpResponseForbidden("You don't have permission to access this view")
+def submit_initial_charter(request, group_type=None, acronym=None):
+
+    # This needs refactoring.
+    # The signature assumed you could have groups with the same name, but with different types, which we do not allow.
+    # Consequently, this can be called with an existing group acronym and a type 
+    # that doesn't match the existing group type. The code below essentially ignores the group_type argument.
+    #
+    # If possible, the use of get_or_create_initial_charter should be moved
+    # directly into charter_submit, and this function should go away.
+
+    if acronym==None:
+        raise Http404
 
     group = get_object_or_404(Group, acronym=acronym)
     if not group.features.has_chartering_process:
         raise Http404
 
+    # This is where we start ignoring the passed in group_type
+    group_type = group.type_id
+
+    if not can_manage_group_type(request.user, group_type):
+        return HttpResponseForbidden("You don't have permission to access this view")
+
     if not group.charter:
         group.charter = get_or_create_initial_charter(group, group_type)
         group.save()

ietf/group/features.py

@@ -20,6 +20,7 @@ def __init__(self, group):
             self.default_tab = "group_docs"
         elif group.type_id in ("team",):
             self.has_materials = True
+            self.default_tab = "group_about"
 
         if self.has_chartering_process:
             self.about_page = "group_charter"