Added a trigger step to the test suite for trunk, triggering the test crawler. Changed the test crawler to run daily at a time which does not overlap the database reload, to avoid spurious errors. Added a step to run migrations to the test crawler. Added a step to the minimum libs test to remove earlier edited requirements before doing svn update.

 - Legacy-Id: 11960

Author: buildbot

buildbot/masters/datatracker/master.cfg

@@ -55,7 +55,7 @@ c['change_source'] = [
 
 from buildbot.schedulers.basic import SingleBranchScheduler, AnyBranchScheduler
 from buildbot.schedulers.forcesched import ForceScheduler
-from buildbot.schedulers.timed import Nightly
+from buildbot.schedulers.timed import Nightly, NightlyTriggerable
 from buildbot.changes import filter
 c['schedulers'] = [
     # Branch schedulers
@@ -65,8 +65,6 @@ c['schedulers'] = [
         change_filter=filter.ChangeFilter(branch='trunk')),
     SingleBranchScheduler(name="osx_test",      treeStableTimer=60*5,   builderNames=["Test Suite (OS X)"],
         change_filter=filter.ChangeFilter(branch='trunk')),
-    SingleBranchScheduler(name="crawler",   treeStableTimer=60*60*4,    builderNames=["Test-Crawler"],
-        change_filter=filter.ChangeFilter(branch='trunk')),
     #
     AnyBranchScheduler(name="pyflakes_branch",  treeStableTimer=10,     builderNames=["[branch] Check PyFlakes"],
         change_filter=filter.ChangeFilter(branch_re='branch/.*')),
@@ -82,8 +80,10 @@ c['schedulers'] = [
     AnyBranchScheduler(name="osx_test_personal",treeStableTimer=60*5,   builderNames=["[personal] Test Suite (OS X)"],
         change_filter=filter.ChangeFilter(branch_re='personal/.*')),
     # Periodic Schedulers
-    Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk",   builderNames=["Verify Latest Libs"],),
-    Nightly(name="lin_test_old_libs", hour=16, minute=42, branch="trunk",   builderNames=["Verify Minimum Libs"],),
+    Nightly(name="lin_test_old_libs",  hour=16, minute=12, branch="trunk",  builderNames=["Verify Minimum Libs"],),
+    Nightly(name="lin_test_libs",      hour=16, minute=42, branch="trunk",  builderNames=["Verify Latest Libs"],),
+    NightlyTriggerable(name="crawler", hour=17, minute=00,                  builderNames=["Test-Crawler"],),
+
     # Force schedulers
     ForceScheduler(name="Force PyFlakes",                               builderNames=["Check PyFlakes"]),
     ForceScheduler(name="Force Test Suite",                             builderNames=["Test Suite"]),
@@ -111,6 +111,7 @@ from buildbot.steps.source.svn import SVN
 from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand
 from buildbot.steps.python import PyFlakes
 from buildbot.steps.python_twisted import RemovePYCs
+from buildbot.steps.trigger import Trigger
 #
 from buildbot.process.properties import Property, Interpolate
 from buildbot.config import BuilderConfig
@@ -123,7 +124,7 @@ class TestCrawlerShellCommand(WarningCountingShellCommand):
     flunkOnFailure = 1
     descriptionDone = ["test crawler"]
     command=["bin/test-crawl"]
-    warningPattern = '.* FAIL'
+    warningPattern = '.*  FAIL'
 
 class UnitTest(WarningCountingShellCommand):
 
@@ -282,21 +283,29 @@ factory.addStep(UnitTest(
                 haltOnFailure=True,
                 command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
             ))
-# This should be the last action
+# This should come after tests
 factory.addStep(ShellCommand(
                 descriptionDone="mark as passed",
                 workdir=Interpolate('build/%(src::branch)s'), 
                 command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
                     "propset", "--revprop", "-r", Property('got_revision'), "test:unittest", "passed" ],
             ))
 
-c['builders'].append(BuilderConfig(name="Test Suite", factory=factory, category="1. trunk",
-        slavenames=["datatracker_lin_py27_1", "datatracker_lin_py27_2", "datatracker_lin_py27_3"]))
 c['builders'].append(BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch",
         slavenames=["datatracker_lin_py27_1", "datatracker_lin_py27_2", "datatracker_lin_py27_3"]))
 c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal",
         slavenames=["datatracker_lin_py27_1", "datatracker_lin_py27_2", "datatracker_lin_py27_3"]))
 
+# For the trunk tests suite only, add a trigger to the test crawler
+# to the steps above
+factory.addStep(Trigger(schedulerNames=['crawler'],
+                        waitForFinish=False,
+                        updateSourceStamp=False,
+                        set_properties={},
+                    ))
+
+c['builders'].append(BuilderConfig(name="Test Suite", factory=factory, category="1. trunk",
+        slavenames=["datatracker_lin_py27_1", "datatracker_lin_py27_2", "datatracker_lin_py27_3"]))
 
 # -*- section Builder_TestSuiteOSX -*-
 
@@ -374,6 +383,12 @@ factory.addStep(ShellCommand(
                 haltOnFailure=True,
                 command=["cp", "/home/buildbot/settings_local.py", "./"],
             ))
+factory.addStep(ShellCommand(
+                descriptionDone="run migrations",
+                workdir=Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["ietf/manage.py", "migrate"], 
+            ))
 factory.addStep(TestCrawlerShellCommand(
                 workdir=Interpolate('build/%(src::branch)s'),
                 haltOnFailure=True,
@@ -400,17 +415,25 @@ c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, categor
 # dependencies.
 
 factory = BuildFactory()
+factory.addStep(ShellCommand(
+                descriptionDone="remove tweaked requirements",
+                workdir=Interpolate('build/%(src::branch)s'),
+                haltOnFailure=False,
+                flunkOnFailure=False,
+                command=["rm", "requirements.txt"],
+            ))
 factory.addStep(SVN(
                 username='buildbot@tools.ietf.org',
                 descriptionDone="svn update",
                 workdir=Interpolate('build/%(src::branch)s'),
+                alwaysUseLatest=True,
                 haltOnFailure=True,
                 repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
                 descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
             ))
 factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
 factory.addStep(ShellCommand(
-                descriptionDone="install/upgrade requirements",
+                descriptionDone="edit requirements",
                 workdir=Interpolate('build/%(src::branch)s'),
                 haltOnFailure=True,
                 command=["sed", "-i", "-e", "s/>=/==/", "requirements.txt"],
@@ -462,6 +485,7 @@ factory.addStep(SVN(
                 username='buildbot@tools.ietf.org',
                 descriptionDone="svn update",
                 workdir=Interpolate('build/%(src::branch)s'),
+                alwaysUseLatest=True,
                 haltOnFailure=True,
                 repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
                 descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],