Skip to content

Commit 73144a6

Browse files
author
buildbot
committed
Updated the buildbot config with additional builders, database loading as part of test-crawler run, tweaked test-crawler start-time and other tweaks
 - Legacy-Id: 17765
1 parent 695b6e0 commit 73144a6

1 file changed

Lines changed: 56 additions & 33 deletions

File tree

‎buildbot/masters/datatracker/master.cfg‎

Lines changed: 56 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,16 @@ c = BuildmasterConfig = {}
2020
# slave name and password must be configured on the slave.
2121
from buildbot.buildslave import BuildSlave
2222
c['slaves'] = [
23-
BuildSlave("datatracker_lin_py27_1", datatracker_lin_py27_1_pw),
24-
BuildSlave("datatracker_lin_py27_2", datatracker_lin_py27_2_pw),
25-
BuildSlave("datatracker_lin_py27_3", datatracker_lin_py27_3_pw),
26-
BuildSlave("datatracker_osx_py27_4", datatracker_osx_py27_4_pw),
27-
BuildSlave("datatracker_lin_py27_5", datatracker_lin_py27_5_pw),
28-
BuildSlave("datatracker_lin_py27_6", datatracker_lin_py27_6_pw),
2923
#
30-
BuildSlave("datatracker_lin_py36_1", datatracker_lin_py36_1_pw),
31-
BuildSlave("datatracker_lin_py36_2", datatracker_lin_py36_2_pw),
32-
BuildSlave("datatracker_lin_py36_3", datatracker_lin_py36_3_pw),
33-
BuildSlave("datatracker_lin_py36_4", datatracker_lin_py36_4_pw),
34-
BuildSlave("datatracker_lin_py36_5", datatracker_lin_py36_5_pw),
35-
BuildSlave("datatracker_lin_py36_6", datatracker_lin_py36_6_pw),
24+
BuildSlave("dunkelfelder_lin_py36_1", dunkelfelder_lin_py36_1_pw),
25+
BuildSlave("dunkelfelder_lin_py36_2", dunkelfelder_lin_py36_2_pw),
26+
BuildSlave("dunkelfelder_lin_py36_3", dunkelfelder_lin_py36_3_pw),
27+
BuildSlave("dunkelfelder_lin_py36_4", dunkelfelder_lin_py36_4_pw),
28+
29+
BuildSlave("dornfelder_lin_py36_1", dornfelder_lin_py36_1_pw),
30+
BuildSlave("dornfelder_lin_py36_2", dornfelder_lin_py36_2_pw),
31+
BuildSlave("dornfelder_lin_py36_3", dornfelder_lin_py36_3_pw),
32+
BuildSlave("dornfelder_lin_py36_4", dornfelder_lin_py36_4_pw),
3633
]
3734

3835
# 'protocols' contains information about protocols which master will use for
@@ -93,7 +90,7 @@ c['schedulers'] = [
9390
# Periodic Schedulers
9491
Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],),
9592
Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],),
96-
Nightly(name="crawler", hour=9, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),
93+
Nightly(name="crawler", hour=23, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),
9794

9895
# Force schedulers
9996
ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]),
@@ -311,6 +308,7 @@ c['builders'] = []
311308
# -*- section Builder_Run_pyflakes -*-
312309

313310
factory = BuildFactory()
311+
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
314312
factory.addStep(SVN(
315313
username='buildbot@tools.ietf.org',
316314
descriptionDone="svn update",
@@ -320,7 +318,13 @@ factory.addStep(SVN(
320318
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
321319
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
322320
))
323-
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
321+
factory.addStep(ShellCommand(
322+
descriptionDone="install requirements",
323+
workdir=Interpolate('build/%(src::branch)s'),
324+
haltOnFailure=True,
325+
usePTY=False,
326+
command=["pip", "install", "-r", "requirements.txt"],
327+
))
324328
factory.addStep(ShellCommand(
325329
descriptionDone="seting up settings_local.py",
326330
workdir=Interpolate('build/%(src::branch)s'),
@@ -338,21 +342,23 @@ factory.addStep(PyFlakes(
338342
factory.addStep(ShellCommand(
339343
descriptionDone="mark as passed",
340344
workdir=Interpolate('build/%(src::branch)s'),
345+
flunkOnFailure=False,
341346
usePTY=False,
342347
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
343348
"propset", "--revprop", "-r", Property('got_revision'), "test:pyflakes", "passed" ],
344349
))
345350

346351
c['builders'].append(BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk",
347-
slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ]))
352+
slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ]))
348353
c['builders'].append(BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch",
349-
slavenames=["datatracker_lin_py36_2", ]))
354+
slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
350355
c['builders'].append(BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal",
351-
slavenames=["datatracker_lin_py36_3", ]))
356+
slavenames=["dunkelfelder_lin_py36_2",]))
352357

353358
# -*- section Builder_TestSuite -*-
354359

355360
factory = BuildFactory()
361+
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
356362
factory.addStep(SVN(
357363
username='buildbot@tools.ietf.org',
358364
descriptionDone="svn update",
@@ -362,7 +368,7 @@ factory.addStep(SVN(
362368
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
363369
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
364370
))
365-
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
371+
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
366372
factory.addStep(ShellCommand(
367373
descriptionDone="remove tmp-* dirs",
368374
workdir=Interpolate('build/%(src::branch)s'),
@@ -377,7 +383,6 @@ factory.addStep(ShellCommand(
377383
usePTY=False,
378384
command=["pip", "install", "-r", "requirements.txt"],
379385
))
380-
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
381386
factory.addStep(ShellCommand(
382387
descriptionDone="copy settings_local.py",
383388
workdir=Interpolate('build/%(src::branch)s'),
@@ -403,22 +408,32 @@ factory.addStep(UnitTest(
403408
factory.addStep(ShellCommand(
404409
descriptionDone="mark as passed",
405410
workdir=Interpolate('build/%(src::branch)s'),
411+
flunkOnFailure=False,
406412
usePTY=False,
407413
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
408414
"propset", "--revprop", "-r", Property('got_revision'), "test:unittest", "passed" ],
409415
))
410416

411417
c['builders'].append(BuilderConfig(name="Test Suite", factory=factory, category="1. trunk",
412-
slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ]))
418+
slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ]))
413419
c['builders'].append(BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch",
414-
slavenames=["datatracker_lin_py36_2", ]))
420+
slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
415421
c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal",
416-
slavenames=["datatracker_lin_py36_3", ]))
422+
slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
417423

418424

419425
# -*- section Builder_TestCrawler -*-
420426

421427
factory = BuildFactory()
428+
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
429+
factory.addStep(ShellCommand(
430+
descriptionDone="update database",
431+
workdir=Interpolate('build/%(src::branch)s'),
432+
haltOnFailure=True,
433+
usePTY=False,
434+
timeout=3600, # 1 hour
435+
command=["docker/updatedb", "-q"],
436+
))
422437
factory.addStep(SVN(
423438
username='buildbot@tools.ietf.org',
424439
descriptionDone="svn update",
@@ -428,15 +443,14 @@ factory.addStep(SVN(
428443
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
429444
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
430445
))
431-
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
446+
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
432447
factory.addStep(ShellCommand(
433448
descriptionDone="install requirements",
434449
workdir=Interpolate('build/%(src::branch)s'),
435450
haltOnFailure=True,
436451
usePTY=False,
437452
command=["pip", "install", "-r", "requirements.txt"],
438453
))
439-
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
440454
factory.addStep(ShellCommand(
441455
descriptionDone="copy settings_local.py",
442456
workdir=Interpolate('build/%(src::branch)s'),
@@ -451,6 +465,14 @@ factory.addStep(ShellCommand(
451465
usePTY=False,
452466
command=["ietf/manage.py", "migrate"],
453467
))
468+
# This will not only do a prelimnary sanity check, but also patch libs as needed:
469+
factory.addStep(ShellCommand(
470+
descriptionDone="run django checks",
471+
workdir=Interpolate('build/%(src::branch)s'),
472+
haltOnFailure=True,
473+
usePTY=False,
474+
command=["ietf/manage.py", "check"],
475+
))
454476
factory.addStep(TestCrawlerShellCommand(
455477
workdir=Interpolate('build/%(src::branch)s'),
456478
haltOnFailure=True,
@@ -461,13 +483,14 @@ factory.addStep(TestCrawlerShellCommand(
461483
factory.addStep(ShellCommand(
462484
descriptionDone="mark as passed",
463485
workdir=Interpolate('build/%(src::branch)s'),
486+
flunkOnFailure=False,
464487
usePTY=False,
465488
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
466489
"propset", "--revprop", "-r", Property('got_revision'), "test:crawler", "passed" ],
467490
))
468491

469492
c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk",
470-
slavenames=["datatracker_lin_py36_6", ]))
493+
slavenames=["dunkelfelder_lin_py36_4", ]))
471494

472495

473496
# -*- section Builder_Verify_Old_Libs -*-
@@ -479,6 +502,7 @@ c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, categor
479502
# dependencies.
480503

481504
factory = BuildFactory()
505+
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
482506
factory.addStep(ShellCommand(
483507
descriptionDone="remove tweaked requirements",
484508
workdir=Interpolate('build/%(src::branch)s'),
@@ -497,7 +521,7 @@ factory.addStep(SVN(
497521
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
498522
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
499523
))
500-
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
524+
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
501525
factory.addStep(ShellCommand(
502526
descriptionDone="edit requirements",
503527
workdir=Interpolate('build/%(src::branch)s'),
@@ -512,7 +536,6 @@ factory.addStep(ShellCommand(
512536
usePTY=False,
513537
command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
514538
))
515-
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
516539
factory.addStep(ShellCommand(
517540
descriptionDone="seting up settings_local.py",
518541
workdir=Interpolate('build/%(src::branch)s'),
@@ -542,10 +565,10 @@ factory.addStep(UnitTest(
542565
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
543566
))
544567
c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk",
545-
slavenames=["datatracker_lin_py36_5", ]))
568+
slavenames=["dornfelder_lin_py36_3", ]))
546569

547570

548-
# -*- section Builder_Dependencies -*-
571+
# -*- section Verify_Latest_Libs -*-
549572

550573
# This build runs pip install --upgrade, to make sure that we install the latest version of all
551574
# dependencies, in order to get an indication if/when an incompatibility turns up with a new
@@ -554,6 +577,7 @@ c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory,
554577
# dependencies.
555578

556579
factory = BuildFactory()
580+
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
557581
factory.addStep(SVN(
558582
username='buildbot@tools.ietf.org',
559583
descriptionDone="svn update",
@@ -564,15 +588,14 @@ factory.addStep(SVN(
564588
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
565589
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
566590
))
567-
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
591+
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
568592
factory.addStep(ShellCommand(
569593
descriptionDone="install/upgrade requirements",
570594
workdir=Interpolate('build/%(src::branch)s'),
571595
haltOnFailure=True,
572596
usePTY=False,
573597
command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
574598
))
575-
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
576599
factory.addStep(ShellCommand(
577600
descriptionDone="seting up settings_local.py",
578601
workdir=Interpolate('build/%(src::branch)s'),
@@ -603,7 +626,7 @@ factory.addStep(UnitTest(
603626
))
604627

605628
c['builders'].append(BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk",
606-
slavenames=["datatracker_lin_py36_5", ]))
629+
slavenames=["dornfelder_lin_py36_3", ]))
607630

608631

609632
####### STATUS TARGETS
@@ -673,7 +696,7 @@ c['status'].append(mail.MailNotifier(
673696
# installation's html.WebStatus home page (linked to the
674697
# 'titleURL') and is embedded in the title of the waterfall HTML page.
675698

676-
c['title'] = "IETF Datatracker"
699+
c['title'] = "Buildbot: IETF Datatracker"
677700
c['titleURL'] = "https://datatracker.ietf.org/"
678701

679702
# the 'buildbotURL' string should point to the location where the buildbot's

0 commit comments

Comments
 (0)