master.cfg

# Copyright The IETF Trust 2015-2020, All Rights Reserved
# -*- python; coding: utf-8 -*-

# This is a buildbot config file for buildbot 0.8.14.p1 (patched to work with
# workers of version 2.7 in addition to 0.8 workers).


import re
from buildbot_passwords import *


# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

# -*- section BuildSlaves -*-

# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.  The same
# slave name and password must be configured on the slave.
from buildbot.buildslave import BuildSlave
c['slaves'] = [
#
    BuildSlave("dunkelfelder_lin_py36_1", dunkelfelder_lin_py36_1_pw),
    BuildSlave("dunkelfelder_lin_py36_2", dunkelfelder_lin_py36_2_pw),
    BuildSlave("dunkelfelder_lin_py36_3", dunkelfelder_lin_py36_3_pw),
    BuildSlave("dunkelfelder_lin_py36_4", dunkelfelder_lin_py36_4_pw),

    BuildSlave("dornfelder_lin_py36_1", dornfelder_lin_py36_1_pw),
    BuildSlave("dornfelder_lin_py36_2", dornfelder_lin_py36_2_pw),
    BuildSlave("dornfelder_lin_py36_3", dornfelder_lin_py36_3_pw),
    BuildSlave("dornfelder_lin_py36_4", dornfelder_lin_py36_4_pw),
]

# 'protocols' contains information about protocols which master will use for
# communicating with slaves.
# You must define at least 'port' option that slaves could connect to your master
# with this protocol.
# 'port' must match the value configured into the buildslaves (with their
# --master option)
c['protocols'] = {'pb': {'host':'zinfandel.tools.ietf.org', 'port': 9989}}

####### CHANGESOURCES
# -*- section ChangeSources -*-

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.

from buildbot.changes.pb import PBChangeSource
# c['change_source'] = []
# with open("users") as file:
#     userinfo = json.read(file)
# for user in userinfo:
#     prefix = userinfo[user]["prefix"]
#     c.['change_source'].append(PBChangeSource(user=user, passwd="BRiR6XcT7x3$", prefix=prefix))
c['change_source'] = [
    PBChangeSource(user="ietfdb", passwd=ietfdb_svn_hook_pw),
]

####### SCHEDULERS
# -*- section Schedulers -*-

# Configure the Schedulers, which decide how to react to incoming changes.  In this
# case, just kick off a 'runtests' build

from buildbot.schedulers.basic import SingleBranchScheduler, AnyBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.schedulers.timed import Nightly
from buildbot.changes import filter
c['schedulers'] = [
    # Branch schedulers
    SingleBranchScheduler(name="pyflakes",      treeStableTimer=10,     builderNames=["Check PyFlakes"],
        change_filter=filter.ChangeFilter(branch='trunk')),
    SingleBranchScheduler(name="lin_test",      treeStableTimer=60*5,   builderNames=["Test Suite"],
        change_filter=filter.ChangeFilter(branch='trunk')),
#    SingleBranchScheduler(name="osx_test",      treeStableTimer=60*5,   builderNames=["Test Suite (OS X)"],
#        change_filter=filter.ChangeFilter(branch='trunk')),
    #
    AnyBranchScheduler(name="pyflakes_branch",  treeStableTimer=10,     builderNames=["[branch] Check PyFlakes"],
        change_filter=filter.ChangeFilter(branch_re='branch/.*')),
#    AnyBranchScheduler(name="lin_test_branch",  treeStableTimer=60*5,   builderNames=["[branch] Test Suite"],
#        change_filter=filter.ChangeFilter(branch_re='branch/.*')),
#    AnyBranchScheduler(name="osx_test_branch",  treeStableTimer=60*5,   builderNames=["[branch] Test Suite (OS X)"],
#        change_filter=filter.ChangeFilter(branch_re='branch/.*')),
    #
    AnyBranchScheduler(name="pyflakes_personal",treeStableTimer=10,     builderNames=["[personal] Check PyFlakes"],
        change_filter=filter.ChangeFilter(branch_re='personal/.*')),
    AnyBranchScheduler(name="lin_test_personal",treeStableTimer=60*5,   builderNames=["[personal] Test Suite"],
        change_filter=filter.ChangeFilter(branch_re='personal/.*')),
    # Periodic Schedulers
    Nightly(name="lin_test_old_libs",  hour=16, minute=12, branch="trunk",  builderNames=["Verify Minimum Libs"],),
    Nightly(name="lin_test_libs",      hour=16, minute=42, branch="trunk",  builderNames=["Verify Latest Libs"],),
    Nightly(name="crawler",            hour=23,  minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),

    # Force schedulers
    ForceScheduler(name="force_pyflakes",                               builderNames=["Check PyFlakes"]),
    ForceScheduler(name="force_lin_test",                               builderNames=["Test Suite"]),
#    ForceScheduler(name="force_osx_test",                               builderNames=["Test Suite (OS X)"]),
    ForceScheduler(name="force_test_crawler",                           builderNames=["Test-Crawler"]),
    #
    ForceScheduler(name="force_pyflakes_branch",                        builderNames=["[branch] Check PyFlakes"]),
    ForceScheduler(name="force_lin_test_branch",                        builderNames=["[branch] Test Suite"]),
#    ForceScheduler(name="force_osx_test_branch",                        builderNames=["[branch] Test Suite (OS X)"]),
    #
    ForceScheduler(name="force_pyflakes_personal",                      builderNames=["[personal] Check PyFlakes"]),
    ForceScheduler(name="force_lin_test_personal",                      builderNames=["[personal] Test Suite"]),
]

####### BUILDERS
# -*- section Builders -*-

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them.  Note that any particular build will
# only take place on one slave.

from buildbot.process.factory import BuildFactory
from buildbot.steps.source.svn import SVN
from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand
from buildbot.steps.python import PyFlakes
from buildbot.steps.python_twisted import RemovePYCs
from buildbot.steps.slave import SetPropertiesFromEnv
#
from buildbot.process.properties import Property, Interpolate
from buildbot.config import BuilderConfig

#### Custom subclassed builder

class TestCrawlerShellCommand(WarningCountingShellCommand):
    name = "testcrawl"
    haltOnFailure = 1
    flunkOnFailure = 1
    descriptionDone = ["test crawler"]
    command=["bin/test-crawl"]

    warningPatterns = {
        "exceptions":       "^(Traceback|  File|    |.*Error|.*Exception)",
        "failed":           " FAIL  ",
        "warnings":         " WARN",
        "slow":             " SLOW",
        "invalid_html":     " invalid html:",
    }

    logline = "^ *(?P<elapsed>\d+:\d+:\d+) +(?P<pages>\d+) +(?P<queue>\d+) +(?P<result>\d+) +(?P<runtime>\d+.\d+)s +(?P<message>.+)"
    
    def setTestResults(self, **kwargs):
        """
        Called by subclasses to set the relevant statistics; this actually
        adds to any statistics already present
        """
        for kw in kwargs:
            value = kwargs[kw]
            if value.isdigit():
                # Counter
                value = int(value)
                value += self.step_status.getStatistic(kw, 0)
            elif re.search("^[0-9]+\.[0-9]+$", value):
                # Runtime
                value = float(value)
                value += self.step_status.getStatistic(kw, 0)
            else:
                # This is a percentage, and we can't add them
                pass                    
            self.step_status.setStatistic(kw, value)

    def createSummary(self, log):
        """
        Match log lines against warningPattern.

        Warnings are collected into another log for this step, and the
        build-wide 'warnings-count' is updated."""

        warnings = {}
        wregex = {}

        regex_class = re.compile("").__class__

        if not isinstance(self.logline, regex_class):
            self.logline = re.compile(self.logline)

        for key in self.warningPatterns:
            warnings[key] = []
            pattern = self.warningPatterns[key]
            if not isinstance(pattern, regex_class):
                wregex[key] = re.compile(pattern)
            else:
                wregex[key] = pattern

        # Count matches to the various warning patterns
        for line in log.getText().split("\n"):
            for key in wregex:
                match = re.search(wregex[key], line)
                if match:
                    warnings[key].append(line)
            if re.search(self.logline, line):
                last_line = line

        # If there were any warnings, make the log if lines with warnings
        # available
        for key in warnings:
            if len(warnings[key]) > 0:
                self.addCompleteLog("%s (%d)" % (key, len(warnings[key])),
                        "\n".join(warnings[key]) + "\n")
                self.step_status.setStatistic(key, len(warnings[key]))
            self.setProperty(key, len(warnings[key]), "TestCrawlerShellCommand")

        match = re.search(self.logline, last_line)
        for key in ['elapsed', 'pages']:
            info = match.group(key)
            self.step_status.setStatistic(key, info)
            self.setProperty(key, info, "TestCrawlerShellCommand")

    def describe(self, done=False):
        description = WarningCountingShellCommand.describe(self, done)
        if done:
            description = description[:]  # make a private copy
            for name in ["time", "elapsed", "pages", "failed", "warnings", "slow", "invalid_html", ]:
                if name in self.step_status.statistics:
                    value = self.step_status.getStatistic(name)
                    displayName = name.replace('_', ' ')
                    # special case. Mph.
                    if   type(value) is float: # this is run-time
                        description.append('%s: %.2fs' % (displayName, value))
                    elif type(value) is int:
                        description.append('%s: %d' % (displayName, value))
                    else:
                        description.append('%s: %s' % (displayName, value))
        return description


class UnitTest(WarningCountingShellCommand):

    name = "test"
    warnOnFailure = 1
    description = ["testing"]
    descriptionDone = ["test"]
    command = ["python", "-m", "unittest", "discover"]

    regexPatterns = {
        "tests":            "Ran (\d+) tests in [0-9.]+s",
        "time":             "Ran \d+ tests in ([0-9.]+)s",
        "skipped":          "(?:OK|FAILED).*skipped=(\d+)",
        "failed":           "FAILED.*failures=(\d+)",
        "errors":           "FAILED.*errors=(\d+)",
        "template_coverage":" +Template coverage: +([0-9.]+%)",
        "url_coverage":     " +Url coverage: +([0-9.]+%)",
        "code_coverage":    " +Code coverage: +([0-9.]+%)",
    }

    def setTestResults(self, **kwargs):
        """
        Called by subclasses to set the relevant statistics; this actually
        adds to any statistics already present
        """
        for kw in kwargs:
            value = kwargs[kw]
            if value.isdigit():
                # Counter
                value = int(value)
                value += self.step_status.getStatistic(kw, 0)
            elif re.search("^[0-9]+\.[0-9]+$", value):
                # Runtime
                value = float(value)
                value += self.step_status.getStatistic(kw, 0)
            else:
                # This is a percentage, and we can't add them
                pass                    
            self.step_status.setStatistic(kw, value)

    def createSummary(self, log):
        info = {}
        for line in log.getText().split("\n"):
            for key in self.regexPatterns:
                regex = self.regexPatterns[key]
                match = re.search(regex, line)
                if match:
                    info[key] = match.group(1)
        self.setTestResults(**info)

    def describe(self, done=False):
        description = WarningCountingShellCommand.describe(self, done)
        if done:
            description = description[:]  # make a private copy
            self.step_status.statistics["passed"] = (
                self.step_status.getStatistic("tests",0) -
                self.step_status.getStatistic("skipped",0) -
                self.step_status.getStatistic("failed",0) -
                self.step_status.getStatistic("errors",0))
            for name in ["time", "tests", "passed", "skipped", "failed", "errors", "template_coverage", "url_coverage", "code_coverage", ]:
                if name in self.step_status.statistics:
                    value = self.step_status.getStatistic(name)
                    displayName = name.replace('_', ' ')
                    # special case. Mph.
                    if displayName == 'template coverage':
                        displayName = 'templ. coverage'
                    if   type(value) is float: # this is run-time
                        description.append('%s: %.2fs' % (displayName, value))
                    elif type(value) is int:
                        description.append('%s: %d' % (displayName, value))
                    else:
                        description.append('%s: %s' % (displayName, value))
        return description


## Set up builders

c['builders'] = []

# -*- section Builder_Run_pyflakes -*-

factory = BuildFactory()
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(SVN(
                username='buildbot@tools.ietf.org',
                descriptionDone="svn update",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
                descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
            ))
factory.addStep(ShellCommand(
                descriptionDone="install requirements",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["pip", "install", "-r", "requirements.txt"],
            ))
factory.addStep(ShellCommand(
                descriptionDone="seting up settings_local.py",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
            ))
factory.addStep(PyFlakes(
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["ietf/manage.py", "pyflakes", "--verbosity=0"],
            ))
# This should be the last action
factory.addStep(ShellCommand(
                descriptionDone="mark as passed",
                workdir=Interpolate('build/%(src::branch)s'), 
                flunkOnFailure=False,
                usePTY=False,
                command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
                    "propset", "--revprop", "-r", Property('got_revision'), "test:pyflakes", "passed" ],
            ))

c['builders'].append(BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk",
                slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ]))
c['builders'].append(BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch",
                slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
c['builders'].append(BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal",
                slavenames=["dunkelfelder_lin_py36_2",]))

# -*- section Builder_TestSuite -*-

factory = BuildFactory()
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(SVN(
                username='buildbot@tools.ietf.org',
                descriptionDone="svn update",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
                descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
            ))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
factory.addStep(ShellCommand(
                descriptionDone="remove tmp-* dirs",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["rm", "-rf", "tmp-*/"],
            ))
factory.addStep(ShellCommand(
                descriptionDone="install requirements",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["pip", "install", "-r", "requirements.txt"],
            ))
factory.addStep(ShellCommand(
                descriptionDone="copy settings_local.py",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
            ))
factory.addStep(ShellCommand(
                descriptionDone="collect static files",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=False,
                flunkOnFailure=False,
                usePTY=False,
                command=["ietf/manage.py", "collectstatic", "--noinput", ],
            ))
factory.addStep(UnitTest(
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
            ))
# This should come after tests
factory.addStep(ShellCommand(
                descriptionDone="mark as passed",
                workdir=Interpolate('build/%(src::branch)s'), 
                flunkOnFailure=False,
                usePTY=False,
                command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
                    "propset", "--revprop", "-r", Property('got_revision'), "test:unittest", "passed" ],
            ))

c['builders'].append(BuilderConfig(name="Test Suite", factory=factory, category="1. trunk",
        slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ]))
c['builders'].append(BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch",
        slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal",
        slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))


# -*- section Builder_TestCrawler -*-

factory = BuildFactory()
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
                descriptionDone="update database",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                timeout=3600,           # 1 hour
                command=["docker/updatedb", "-q"],
            ))
factory.addStep(SVN(
                username='buildbot@tools.ietf.org',
                descriptionDone="svn update",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
                descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
            ))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
factory.addStep(ShellCommand(
                descriptionDone="install requirements",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["pip", "install", "-r", "requirements.txt"],
            ))
factory.addStep(ShellCommand(
                descriptionDone="copy settings_local.py",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
            ))
factory.addStep(ShellCommand(
                descriptionDone="run migrations",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["ietf/manage.py", "migrate"], 
            ))
# This will not only do a prelimnary sanity check, but also patch libs as needed:
factory.addStep(ShellCommand(
                descriptionDone="run django checks",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["ietf/manage.py", "check"], 
            ))
factory.addStep(TestCrawlerShellCommand(
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["bin/test-crawl", "--settings=ietf.settings_testcrawl"],
            ))
# This should be the last action
factory.addStep(ShellCommand(
                descriptionDone="mark as passed",
                workdir=Interpolate('build/%(src::branch)s'), 
                flunkOnFailure=False,
                usePTY=False,
                command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
                    "propset", "--revprop", "-r", Property('got_revision'), "test:crawler", "passed" ],
            ))

c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk",
        slavenames=["dunkelfelder_lin_py36_4", ]))


# -*- section Builder_Verify_Old_Libs -*-

# This build runs pip install --upgrade, to make sure that we install the earliest version of
# all dependencies, in order to get an indication if/when an incompatibility turns up with a new
# version of a dependency.  The other test suite builders *don't* use --upgrade, in order to not
# change the external test conditions and produce spurious errors because of version changes in
# dependencies.

factory = BuildFactory()
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
                descriptionDone="remove tweaked requirements",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=False,
                flunkOnFailure=False,
                usePTY=False,
                command=["rm", "requirements.txt"],
            ))
factory.addStep(SVN(
                username='buildbot@tools.ietf.org',
                descriptionDone="svn update",
                workdir=Interpolate('build/%(src::branch)s'),
                alwaysUseLatest=True,
                haltOnFailure=True,
                usePTY=False,
                repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
                descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
            ))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
factory.addStep(ShellCommand(
                descriptionDone="edit requirements",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["sed", "-i", "-e", "s/>=/==/", "requirements.txt"],
            ))
factory.addStep(ShellCommand(
                descriptionDone="install/upgrade requirements",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
            ))
factory.addStep(ShellCommand(
                descriptionDone="seting up settings_local.py",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
            ))
factory.addStep(ShellCommand(
                descriptionDone="list installed pyton modules",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["pip", "freeze"],
            ))
factory.addStep(ShellCommand(
                descriptionDone="collect static files",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=False,
                flunkOnFailure=False,
                usePTY=False,
                command=["ietf/manage.py", "collectstatic", "--noinput", ],
            ))
factory.addStep(UnitTest(
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
            ))
c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk",
        slavenames=["dornfelder_lin_py36_3", ]))


# -*- section Verify_Latest_Libs -*-

# This build runs pip install --upgrade, to make sure that we install the latest version of all
# dependencies, in order to get an indication if/when an incompatibility turns up with a new
# version of a dependency.  The other test suite builders *don't* use --upgrade, in order to not
# change the external test conditions and produce spurious errors because of version changes in
# dependencies.

factory = BuildFactory()
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(SVN(
                username='buildbot@tools.ietf.org',
                descriptionDone="svn update",
                workdir=Interpolate('build/%(src::branch)s'),
                alwaysUseLatest=True,
                haltOnFailure=True,
                usePTY=False,
                repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
                descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
            ))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
factory.addStep(ShellCommand(
                descriptionDone="install/upgrade requirements",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
            ))
factory.addStep(ShellCommand(
                descriptionDone="seting up settings_local.py",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
            ))
factory.addStep(ShellCommand(
                descriptionDone="list installed pyton modules",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["pip", "freeze"],
            ))
factory.addStep(ShellCommand(
                descriptionDone="collect static files",
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=False,
                flunkOnFailure=False,
                usePTY=False,
                command=["ietf/manage.py", "collectstatic", "--noinput", ],
            ))
factory.addStep(UnitTest(
                workdir=Interpolate('build/%(src::branch)s'),
                haltOnFailure=True,
                usePTY=False,
                command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
            ))

c['builders'].append(BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk",
        slavenames=["dornfelder_lin_py36_3", ]))


####### STATUS TARGETS
# -*- section StatusTargets -*-

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

from buildbot.status import html, mail
from buildbot.status.web import authz, auth

authz_cfg=authz.Authz(
    # change any of these to True to enable; see the manual for more
    # options
    auth=auth.BasicAuth([("ietfdb","ietfdb")]),
    gracefulShutdown = False,
    forceBuild = 'auth', # use this to test your slave once it is set up
    forceAllBuilds = False,
    pingBuilder = False,
    stopBuild = 'auth',
    stopAllBuilds = False,
    cancelPendingBuild = 'auth',
)
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))

# A second web status with slightly different rendering
from twisted.python import log
def changelinkfilter(html, project):
    log.msg(" * changelinkfilter(html='%s', project='%s')" % (html, project))
    return html

import jinja2, os
trac_template_loaders = [jinja2.FileSystemLoader(os.path.join(os.getcwd(), 'trac_view'))]
c['status'].append(html.WebStatus(http_port=8011, jinja_loaders=trac_template_loaders,
    authz=authz_cfg))

# Email notifications
from zope.interface import implements
from buildbot import interfaces

class UsernameIsEmailAddress():
    "This IEmailLookup provider assumes that the svn username is a valid email address."
    implements(interfaces.IEmailLookup)
    def getAddress(self, name):
        return name

c['status'].append(mail.MailNotifier(
    fromaddr='buildbot@tools.ietf.org',
    sendToInterestedUsers=False,
    extraRecipients=['henrik@levkowetz.com',],
    mode="problem",
    ))

c['status'].append(mail.MailNotifier(
    fromaddr='buildbot@tools.ietf.org',
    lookup=UsernameIsEmailAddress(),
    mode="problem",
    ))

####### PROJECT IDENTITY
# -*- section Project -*-

# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = "Buildbot: IETF Datatracker"
c['titleURL'] = "https://datatracker.ietf.org/"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = "http://zinfandel.tools.ietf.org:8010/"

####### DB URL

c['db'] = {
    # This specifies what database buildbot uses to store its state.  You can leave
    # this at its default for all but the largest installations.
    'db_url' : "sqlite:///state.sqlite",
}