Merged in ^/trunk@17782 (just ahead of release 6.130.0)

 - Legacy-Id: 17783

Author: levkowetz

bin/daily

@@ -61,4 +61,4 @@ $DTDIR/ietf/manage.py fetch_meeting_attendance --latest 2
 $DTDIR/ietf/bin/send-review-reminders
 
 # Purge old request_profiler records
-$DTDIR/ietf/manage.py purge_request_profiler_records
+$DTDIR/ietf/manage.py purge_request_profiler_records -d2

bin/dump-to-names-json

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# This script provides a limited selected dump of database content with the
+# purpose of generating a test fixture that provides the test data needed
+# by the test suite.
+#
+# The generated data fixture is sorted and normalized in order to produce
+# minimal commit diffs which reflect only actual changes in the fixture data,
+# without apparent changes resulting only from ordering changes.
+
+set -x
+ietf/manage.py dumpdata --indent 1 doc.State doc.BallotType doc.StateType       \
+        mailtrigger.MailTrigger mailtrigger.Recipient name utils.VersionInfo    \
+        group.GroupFeatures stats.CountryAlias dbtemplate.DBTemplate            \
+        | jq --sort-keys "sort_by(.model, .pk)"                                 \
+        | jq '[.[] | select(.model!="dbtemplate.dbtemplate" or .pk==354)]' > ietf/name/fixtures/names.json

bin/every15m

@@ -16,7 +16,5 @@ source $DTDIR/env/bin/activate
 
 logger -p user.info -t cron "Running $DTDIR/bin/every15m"
 
-
-
-
-
+# Send mail scheduled to go out at certain times
+$DTDIR/ietf/bin/send-scheduled-mail all

bin/test-crawl

@@ -138,6 +138,7 @@ def check_html_valid(url, response, args):
         key = re.sub("/dir/[a-z0-9-]+/",   "/dir/foo/", key)
         key = re.sub("/draft-[a-z0-9-]+/",   "/draft-foo/", key)
         key = re.sub("/group/[a-z0-9-]+/",   "/group/foo/", key)
+        key = re.sub("/html/[a-z0-9-]+",   "/html/foo/", key)
         key = re.sub("/ipr/search/.*",   "/ipr/search/", key)
         key = re.sub("/meeting/[-0-9a-z]+/agenda/[0-9a-z]+/",   "/meeting/nn/agenda/foo/", key)
         key = re.sub("/release/[0-9dev.]+/",   "/release/n.n.n/", key)

bin/weekly

@@ -20,3 +20,6 @@ logger -p user.info -t cron "Running $DTDIR/bin/weekly"
 
 $DTDIR/ietf/manage.py send_apikey_usage_emails
 
+# Send notifications about coming expirations
+$DTDIR/ietf/bin/notify-expirations
+

buildbot/masters/datatracker/master.cfg

@@ -20,19 +20,16 @@ c = BuildmasterConfig = {}
 # slave name and password must be configured on the slave.
 from buildbot.buildslave import BuildSlave
 c['slaves'] = [
-    BuildSlave("datatracker_lin_py27_1", datatracker_lin_py27_1_pw),
-    BuildSlave("datatracker_lin_py27_2", datatracker_lin_py27_2_pw),
-    BuildSlave("datatracker_lin_py27_3", datatracker_lin_py27_3_pw),
-    BuildSlave("datatracker_osx_py27_4", datatracker_osx_py27_4_pw),
-    BuildSlave("datatracker_lin_py27_5", datatracker_lin_py27_5_pw),
-    BuildSlave("datatracker_lin_py27_6", datatracker_lin_py27_6_pw),
 #
-    BuildSlave("datatracker_lin_py36_1", datatracker_lin_py36_1_pw),
-    BuildSlave("datatracker_lin_py36_2", datatracker_lin_py36_2_pw),
-    BuildSlave("datatracker_lin_py36_3", datatracker_lin_py36_3_pw),
-    BuildSlave("datatracker_lin_py36_4", datatracker_lin_py36_4_pw),
-    BuildSlave("datatracker_lin_py36_5", datatracker_lin_py36_5_pw),
-    BuildSlave("datatracker_lin_py36_6", datatracker_lin_py36_6_pw),
+    BuildSlave("dunkelfelder_lin_py36_1", dunkelfelder_lin_py36_1_pw),
+    BuildSlave("dunkelfelder_lin_py36_2", dunkelfelder_lin_py36_2_pw),
+    BuildSlave("dunkelfelder_lin_py36_3", dunkelfelder_lin_py36_3_pw),
+    BuildSlave("dunkelfelder_lin_py36_4", dunkelfelder_lin_py36_4_pw),
+
+    BuildSlave("dornfelder_lin_py36_1", dornfelder_lin_py36_1_pw),
+    BuildSlave("dornfelder_lin_py36_2", dornfelder_lin_py36_2_pw),
+    BuildSlave("dornfelder_lin_py36_3", dornfelder_lin_py36_3_pw),
+    BuildSlave("dornfelder_lin_py36_4", dornfelder_lin_py36_4_pw),
 ]
 
 # 'protocols' contains information about protocols which master will use for
@@ -93,7 +90,7 @@ c['schedulers'] = [
     # Periodic Schedulers
     Nightly(name="lin_test_old_libs",  hour=16, minute=12, branch="trunk",  builderNames=["Verify Minimum Libs"],),
     Nightly(name="lin_test_libs",      hour=16, minute=42, branch="trunk",  builderNames=["Verify Latest Libs"],),
-    Nightly(name="crawler",            hour=9,  minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),
+    Nightly(name="crawler",            hour=23,  minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),
 
     # Force schedulers
     ForceScheduler(name="force_pyflakes",                               builderNames=["Check PyFlakes"]),
@@ -311,6 +308,7 @@ c['builders'] = []
 # -*- section Builder_Run_pyflakes -*-
 
 factory = BuildFactory()
+factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
 factory.addStep(SVN(
                 username='buildbot@tools.ietf.org',
                 descriptionDone="svn update",
@@ -320,7 +318,13 @@ factory.addStep(SVN(
                 repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
                 descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
             ))
-factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
+factory.addStep(ShellCommand(
+                descriptionDone="install requirements",
+                workdir=Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                usePTY=False,
+                command=["pip", "install", "-r", "requirements.txt"],
+            ))
 factory.addStep(ShellCommand(
                 descriptionDone="seting up settings_local.py",
                 workdir=Interpolate('build/%(src::branch)s'),
@@ -338,21 +342,23 @@ factory.addStep(PyFlakes(
 factory.addStep(ShellCommand(
                 descriptionDone="mark as passed",
                 workdir=Interpolate('build/%(src::branch)s'), 
+                flunkOnFailure=False,
                 usePTY=False,
                 command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
                     "propset", "--revprop", "-r", Property('got_revision'), "test:pyflakes", "passed" ],
             ))
 
 c['builders'].append(BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk",
-                slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ]))
+                slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ]))
 c['builders'].append(BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch",
-                slavenames=["datatracker_lin_py36_2", ]))
+                slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
 c['builders'].append(BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal",
-                slavenames=["datatracker_lin_py36_3", ]))
+                slavenames=["dunkelfelder_lin_py36_2",]))
 
 # -*- section Builder_TestSuite -*-
 
 factory = BuildFactory()
+factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
 factory.addStep(SVN(
                 username='buildbot@tools.ietf.org',
                 descriptionDone="svn update",
@@ -362,7 +368,7 @@ factory.addStep(SVN(
                 repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
                 descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
             ))
-factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
+factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
 factory.addStep(ShellCommand(
                 descriptionDone="remove tmp-* dirs",
                 workdir=Interpolate('build/%(src::branch)s'),
@@ -377,7 +383,6 @@ factory.addStep(ShellCommand(
                 usePTY=False,
                 command=["pip", "install", "-r", "requirements.txt"],
             ))
-factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
 factory.addStep(ShellCommand(
                 descriptionDone="copy settings_local.py",
                 workdir=Interpolate('build/%(src::branch)s'),
@@ -403,22 +408,32 @@ factory.addStep(UnitTest(
 factory.addStep(ShellCommand(
                 descriptionDone="mark as passed",
                 workdir=Interpolate('build/%(src::branch)s'), 
+                flunkOnFailure=False,
                 usePTY=False,
                 command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
                     "propset", "--revprop", "-r", Property('got_revision'), "test:unittest", "passed" ],
             ))
 
 c['builders'].append(BuilderConfig(name="Test Suite", factory=factory, category="1. trunk",
-        slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ]))
+        slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ]))
 c['builders'].append(BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch",
-        slavenames=["datatracker_lin_py36_2", ]))
+        slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
 c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal",
-        slavenames=["datatracker_lin_py36_3", ]))
+        slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
 
 
 # -*- section Builder_TestCrawler -*-
 
 factory = BuildFactory()
+factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
+factory.addStep(ShellCommand(
+                descriptionDone="update database",
+                workdir=Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                usePTY=False,
+                timeout=3600,           # 1 hour
+                command=["docker/updatedb", "-q"],
+            ))
 factory.addStep(SVN(
                 username='buildbot@tools.ietf.org',
                 descriptionDone="svn update",
@@ -428,15 +443,14 @@ factory.addStep(SVN(
                 repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
                 descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
             ))
-factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
+factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
 factory.addStep(ShellCommand(
                 descriptionDone="install requirements",
                 workdir=Interpolate('build/%(src::branch)s'),
                 haltOnFailure=True,
                 usePTY=False,
                 command=["pip", "install", "-r", "requirements.txt"],
             ))
-factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
 factory.addStep(ShellCommand(
                 descriptionDone="copy settings_local.py",
                 workdir=Interpolate('build/%(src::branch)s'),
@@ -451,6 +465,14 @@ factory.addStep(ShellCommand(
                 usePTY=False,
                 command=["ietf/manage.py", "migrate"], 
             ))
+# This will not only do a prelimnary sanity check, but also patch libs as needed:
+factory.addStep(ShellCommand(
+                descriptionDone="run django checks",
+                workdir=Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                usePTY=False,
+                command=["ietf/manage.py", "check"], 
+            ))
 factory.addStep(TestCrawlerShellCommand(
                 workdir=Interpolate('build/%(src::branch)s'),
                 haltOnFailure=True,
@@ -461,13 +483,14 @@ factory.addStep(TestCrawlerShellCommand(
 factory.addStep(ShellCommand(
                 descriptionDone="mark as passed",
                 workdir=Interpolate('build/%(src::branch)s'), 
+                flunkOnFailure=False,
                 usePTY=False,
                 command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
                     "propset", "--revprop", "-r", Property('got_revision'), "test:crawler", "passed" ],
             ))
 
 c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk",
-        slavenames=["datatracker_lin_py36_6", ]))
+        slavenames=["dunkelfelder_lin_py36_4", ]))
 
 
 # -*- section Builder_Verify_Old_Libs -*-
@@ -479,6 +502,7 @@ c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, categor
 # dependencies.
 
 factory = BuildFactory()
+factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
 factory.addStep(ShellCommand(
                 descriptionDone="remove tweaked requirements",
                 workdir=Interpolate('build/%(src::branch)s'),
@@ -497,7 +521,7 @@ factory.addStep(SVN(
                 repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
                 descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
             ))
-factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
+factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
 factory.addStep(ShellCommand(
                 descriptionDone="edit requirements",
                 workdir=Interpolate('build/%(src::branch)s'),
@@ -512,7 +536,6 @@ factory.addStep(ShellCommand(
                 usePTY=False,
                 command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
             ))
-factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
 factory.addStep(ShellCommand(
                 descriptionDone="seting up settings_local.py",
                 workdir=Interpolate('build/%(src::branch)s'),
@@ -542,10 +565,10 @@ factory.addStep(UnitTest(
                 command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
             ))
 c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk",
-        slavenames=["datatracker_lin_py36_5", ]))
+        slavenames=["dornfelder_lin_py36_3", ]))
 
 
-# -*- section Builder_Dependencies -*-
+# -*- section Verify_Latest_Libs -*-
 
 # This build runs pip install --upgrade, to make sure that we install the latest version of all
 # dependencies, in order to get an indication if/when an incompatibility turns up with a new
@@ -554,6 +577,7 @@ c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory,
 # dependencies.
 
 factory = BuildFactory()
+factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
 factory.addStep(SVN(
                 username='buildbot@tools.ietf.org',
                 descriptionDone="svn update",
@@ -564,15 +588,14 @@ factory.addStep(SVN(
                 repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
                 descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
             ))
-factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
+factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
 factory.addStep(ShellCommand(
                 descriptionDone="install/upgrade requirements",
                 workdir=Interpolate('build/%(src::branch)s'),
                 haltOnFailure=True,
                 usePTY=False,
                 command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
             ))
-factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
 factory.addStep(ShellCommand(
                 descriptionDone="seting up settings_local.py",
                 workdir=Interpolate('build/%(src::branch)s'),
@@ -603,7 +626,7 @@ factory.addStep(UnitTest(
             ))
 
 c['builders'].append(BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk",
-        slavenames=["datatracker_lin_py36_5", ]))
+        slavenames=["dornfelder_lin_py36_3", ]))
 
 
 ####### STATUS TARGETS
@@ -673,7 +696,7 @@ c['status'].append(mail.MailNotifier(
 # installation's html.WebStatus home page (linked to the
 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 
-c['title'] = "IETF Datatracker"
+c['title'] = "Buildbot: IETF Datatracker"
 c['titleURL'] = "https://datatracker.ietf.org/"
 
 # the 'buildbotURL' string should point to the location where the buildbot's

ietf/doc/models.py

@@ -158,7 +158,7 @@ def get_base_name(self):
                     else:
                         self._cached_base_name = "%s-%s.txt" % (self.name, self.rev)
             elif self.type_id in ["slides", "agenda", "minutes", "bluesheets", ] and self.meeting_related():
-                    self._cached_base_name = "%s-%s.txt" % self.canonical_name() 
+                    self._cached_base_name = "%s-%s.txt" % (self.canonical_name(), self.rev) 
             elif self.type_id == 'review':
                 # TODO: This will be wrong if a review is updated on the same day it was created (or updated more than once on the same day)
                 self._cached_base_name = "%s.txt" % self.name

ietf/doc/views_doc.py

@@ -256,7 +256,9 @@ def document_main(request, name, rev=None):
 
             if "pdf" not in found_types:
                 file_urls.append(("pdf", settings.TOOLS_ID_PDF_URL + doc.name + "-" + doc.rev + ".pdf"))
-            file_urls.append(("htmlized", settings.TOOLS_ID_HTML_URL + doc.name + "-" + doc.rev))
+            #file_urls.append(("htmlized", settings.TOOLS_ID_HTML_URL + doc.name + "-" + doc.rev))
+            file_urls.append(("htmlized (tools)", settings.TOOLS_ID_HTML_URL + doc.name + "-" + doc.rev))
+            file_urls.append(("htmlized", urlreverse('ietf.doc.views_doc.document_html', kwargs=dict(name=doc.name, rev=doc.rev))))
 
             # latest revision
             latest_revision = doc.latest_event(NewRevisionDocEvent, type="new_revision")
@@ -683,7 +685,7 @@ def document_html(request, name, rev=None):
 
     doc = docs.get()
     if not os.path.exists(doc.get_file_name()):
-        raise Http404("Document not found: %s" % doc.get_base_name())
+        raise Http404("File not found: %s" % doc.get_file_name())
 
     top = render_document_top(request, doc, "status", name)
     if not rev and not name.startswith('rfc'):

ietf/group/factories.py

@@ -18,6 +18,15 @@ class Meta:
     list_email = factory.LazyAttribute(lambda a: '%s@ietf.org'% a.acronym)
     uses_milestone_dates = True
 
+    @factory.lazy_attribute
+    def parent(self):
+        if self.type_id in ['wg','ag']:
+            return GroupFactory(type_id='area')
+        elif self.type_id in ['rg']:
+            return GroupFactory(acronym='irtf', type_id='irtf')
+        else:
+            return None
+
 class ReviewTeamFactory(GroupFactory):
 
     type_id = 'review'

ietf/group/migrations/0024_add_groupman_authroles.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-05-04 13:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+import jsonfield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('group', '0023_use_milestone_dates_default_to_true'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='groupfeatures',
+            name='groupman_authroles',
+            field=jsonfield.fields.JSONField(default=['Secretariat'], max_length=128),
+        ),
+        migrations.AddField(
+            model_name='historicalgroupfeatures',
+            name='groupman_authroles',
+            field=jsonfield.fields.JSONField(default=['Secretariat'], max_length=128),
+        ),
+    ]