Adding upstream version 2:4.20.0+dfsg.upstream/2%4.20.0+dfsg

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1902 insertions, 0 deletions

diff --git a/script/autobuild.py b/script/autobuild.py
new file mode 100755
index 0000000..ecec352
--- /dev/null
+++ b/script/autobuild.py
@@ -0,0 +1,1902 @@
+#!/usr/bin/env python3
+# run tests on all Samba subprojects and push to a git tree on success
+# Copyright Andrew Tridgell 2010
+# released under GNU GPL v3 or later
+
+from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
+import os
+import tarfile
+import sys
+import time
+import random
+from optparse import OptionParser
+import smtplib
+import email
+from email.mime.text import MIMEText
+from email.mime.base import MIMEBase
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from sysconfig import get_path
+import platform
+
+import logging
+
+try:
+    from waflib.Build import CACHE_SUFFIX
+except ImportError:
+    sys.path.insert(0, "./third_party/waf")
+    from waflib.Build import CACHE_SUFFIX
+
+logging.basicConfig(format='%(asctime)s %(message)s')
+logger = logging.getLogger('autobuild')
+logger.setLevel(logging.INFO)
+
+os.environ["PYTHONUNBUFFERED"] = "1"
+
+# This speeds up testing remarkably.
+os.environ['TDB_NO_FSYNC'] = '1'
+
+# allow autobuild to run within git rebase -i
+if "GIT_DIR" in os.environ:
+    del os.environ["GIT_DIR"]
+if "GIT_WORK_TREE" in os.environ:
+    del os.environ["GIT_WORK_TREE"]
+
+def find_git_root():
+    '''get to the top of the git repo'''
+    p = os.getcwd()
+    while p != '/':
+        if os.path.exists(os.path.join(p, ".git")):
+            return p
+        p = os.path.abspath(os.path.join(p, '..'))
+    return None
+
+
+gitroot = find_git_root()
+if gitroot is None:
+    raise Exception("Failed to find git root")
+
+
+def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
+
+parser = OptionParser()
+parser.add_option("--tail", help="show output while running", default=False, action="store_true")
+parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
+parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
+parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
+parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
+                  default=def_testbase)
+parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
+                  default=None)
+parser.add_option("--passcmd", help="command to run on success", default=None)
+parser.add_option("--verbose", help="show all commands as they are run",
+                  default=False, action="store_true")
+parser.add_option("--rebase", help="rebase on the given tree before testing",
+                  default=None, type='str')
+parser.add_option("--pushto", help="push to a git url on success",
+                  default=None, type='str')
+parser.add_option("--mark", help="add a Tested-By signoff before pushing",
+                  default=False, action="store_true")
+parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
+                  default=False, action="store_true")
+parser.add_option("--retry", help="automatically retry if master changes",
+                  default=False, action="store_true")
+parser.add_option("--email", help="send email to the given address on failure",
+                  type='str', default=None)
+parser.add_option("--email-from", help="send email from the given address",
+                  type='str', default="autobuild@samba.org")
+parser.add_option("--email-server", help="send email via the given server",
+                  type='str', default='localhost')
+parser.add_option("--always-email", help="always send email, even on success",
+                  action="store_true")
+parser.add_option("--daemon", help="daemonize after initial setup",
+                  action="store_true")
+parser.add_option("--branch", help="the branch to work on (default=master)",
+                  default="master", type='str')
+parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
+                  default=gitroot, type='str')
+parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
+                  default=False, action="store_true")
+parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
+                  default='')
+parser.add_option("--enable-coverage", dest='enable_coverage',
+                  action="store_const", const='--enable-coverage', default='',
+                  help="Add --enable-coverage option while configure")
+
+(options, args) = parser.parse_args()
+
+if options.retry:
+    if options.rebase is None:
+        raise Exception('You can only use --retry if you also rebase')
+
+if options.verbose:
+    logger.setLevel(logging.DEBUG)
+
+if options.full_testbase is not None:
+    testbase = options.full_testbase
+else:
+    testbase = "%s/b%u" % (options.testbase, os.getpid())
+test_master = "%s/master" % testbase
+test_prefix = "%s/prefix" % testbase
+test_tmpdir = "%s/tmp" % testbase
+os.environ['TMPDIR'] = test_tmpdir
+
+if options.enable_coverage:
+    LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
+else:
+    LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
+
+if options.enable_coverage:
+    PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
+else:
+    PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
+
+CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
+
+
+def check_symbols(sofile, expected_symbols=""):
+    return "objdump --dynamic-syms " + sofile + " | " + \
+           "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
+
+if args:
+    # If we are only running specific test,
+    # do not sleep randomly to wait for it to start
+    def random_sleep(low, high):
+        return 'sleep 1'
+else:
+    def random_sleep(low, high):
+        return 'sleep {}'.format(random.randint(low, high))
+
+cleanup_list = []
+
+builddirs = {
+    "ctdb": "ctdb",
+    "ldb": "lib/ldb",
+    "tdb": "lib/tdb",
+    "talloc": "lib/talloc",
+    "replace": "lib/replace",
+    "tevent": "lib/tevent",
+    "pidl": "pidl",
+    "docs-xml": "docs-xml"
+}
+
+ctdb_configure_params = " --enable-developer ${PREFIX}"
+samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
+
+samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
+samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
+samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
+samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
+samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
+samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
+samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
+
+
+def format_option(name, value=None):
+    """Format option as str list."""
+    if value is None:  # boolean option
+        return [name]
+    if not isinstance(value, list):  # single value option
+        value = [value]
+    # repeatable option
+    return ['{}={}'.format(name, item) for item in value]
+
+
+def make_test(
+        cmd='make testonly',
+        INJECT_SELFTEST_PREFIX=1,
+        TESTS='',
+        include_envs=None,
+        exclude_envs=None):
+
+    test_options = []
+    if include_envs:
+        test_options = format_option('--include-env', include_envs)
+    if exclude_envs:
+        test_options = format_option('--exclude-env', exclude_envs)
+    if test_options:
+        # join envs options to original test options
+        TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
+
+    _options = []
+
+    # Allow getting a full CI with
+    # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
+
+    FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
+
+    if int(FAIL_IMMEDIATELY):
+        _options.append('FAIL_IMMEDIATELY=1')
+    if TESTS:
+        _options.append("TESTS='{}'".format(TESTS))
+
+    if INJECT_SELFTEST_PREFIX:
+        _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
+        _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
+
+    return ' '.join([cmd] + _options)
+
+
+# When updating this list, also update .gitlab-ci.yml to add the job
+# and to make it a dependency of 'page' for the coverage report.
+
+tasks = {
+    "ctdb": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("configure", "./configure " + ctdb_configure_params),
+            ("make", "make all"),
+            ("install", "make install"),
+            ("test", "make autotest"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+        ],
+    },
+    "docs-xml": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("autoconf", "autoconf"),
+            ("configure", "./configure"),
+            ("make", "make html htmlman"),
+            ("publish-docs", PUBLISH_DOCS),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "samba-def-build": {
+        "git-clone-required": True,
+        "sequence": [
+            ("configure", "./configure.developer" + samba_configure_params),
+            ("make", "make -j"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("chmod-R-a-w", "chmod -R a-w ."),
+        ],
+    },
+
+    "samba-mit-build": {
+        "git-clone-required": True,
+        "sequence": [
+            ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
+            ("make", "make -j"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("chmod-R-a-w", "chmod -R a-w ."),
+        ],
+    },
+
+    "samba-nt4-build": {
+        "git-clone-required": True,
+        "sequence": [
+            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
+            ("make", "make -j"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("chmod-R-a-w", "chmod -R a-w ."),
+        ],
+    },
+
+    "samba-h5l-build": {
+        "git-clone-required": True,
+        "sequence": [
+            ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
+            ("make", "make -j"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("chmod-R-a-w", "chmod -R a-w ."),
+        ],
+    },
+
+    "samba-without-smb1-build": {
+        "git-clone-required": True,
+        "sequence": [
+            ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
+            ("make", "make -j"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("chmod-R-a-w", "chmod -R a-w ."),
+        ],
+    },
+
+    "samba-no-opath-build": {
+        "git-clone-required": True,
+        "sequence": [
+            ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1 -DDISABLE_VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS=1 -DDISABLE_PROC_FDS=1' ./configure.developer --without-ad-dc " + samba_configure_params),
+            ("make", "make -j"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("chmod-R-a-w", "chmod -R a-w ."),
+        ],
+    },
+
+    # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
+    "samba": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("configure", "./configure.developer" + samba_configure_params),
+            ("make", "make -j"),
+            ("test", make_test(exclude_envs=[
+            "none",
+            "nt4_dc",
+            "nt4_dc_smb1",
+            "nt4_dc_smb1_done",
+            "nt4_dc_schannel",
+            "nt4_member",
+            "ad_dc",
+            "ad_dc_smb1",
+            "ad_dc_smb1_done",
+            "ad_dc_backup",
+            "ad_dc_ntvfs",
+            "ad_dc_default",
+            "ad_dc_default_smb1",
+            "ad_dc_slowtests",
+            "ad_dc_no_nss",
+            "ad_dc_no_ntlm",
+            "fl2003dc",
+            "fl2008dc",
+            "fl2008r2dc",
+            "ad_member",
+            "ad_member_idmap_rid",
+            "admem_idmap_autorid",
+            "ad_member_idmap_ad",
+            "ad_member_rfc2307",
+            "ad_member_idmap_nss",
+            "ad_member_oneway",
+            "chgdcpass",
+            "vampire_2000_dc",
+            "fl2000dc",
+            "fileserver",
+            "fileserver_smb1",
+            "fileserver_smb1_done",
+            "maptoguest",
+            "simpleserver",
+            "backupfromdc",
+            "restoredc",
+            "renamedc",
+            "offlinebackupdc",
+            "labdc",
+            "preforkrestartdc",
+            "proclimitdc",
+            "promoted_dc",
+            "vampire_dc",
+            "rodc",
+            "ad_dc_default",
+            "ad_dc_default_smb1",
+            "ad_dc_default_smb1_done",
+            "ad_dc_slowtests",
+            "schema_pair_dc",
+            "schema_dc",
+            "clusteredmember",
+            "ad_dc_fips",
+            "ad_member_fips",
+            ])),
+            ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
+            ("lcov", LCOV_CMD),
+            ("install", "make install"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+        ],
+    },
+
+    # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
+    "samba-mitkrb5": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
+            ("make", "make -j"),
+            ("test", make_test(exclude_envs=[
+            "none",
+            "nt4_dc",
+            "nt4_dc_smb1",
+            "nt4_dc_smb1_done",
+            "nt4_dc_schannel",
+            "nt4_member",
+            "ad_dc",
+            "ad_dc_smb1",
+            "ad_dc_smb1_done",
+            "ad_dc_backup",
+            "ad_dc_ntvfs",
+            "ad_dc_default",
+            "ad_dc_default_smb1",
+            "ad_dc_default_smb1_done",
+            "ad_dc_slowtests",
+            "ad_dc_no_nss",
+            "ad_dc_no_ntlm",
+            "fl2003dc",
+            "fl2008dc",
+            "fl2008r2dc",
+            "ad_member",
+            "ad_member_idmap_rid",
+            "admem_idmap_autorid",
+            "ad_member_idmap_ad",
+            "ad_member_rfc2307",
+            "ad_member_idmap_nss",
+            "ad_member_oneway",
+            "chgdcpass",
+            "vampire_2000_dc",
+            "fl2000dc",
+            "fileserver",
+            "fileserver_smb1",
+            "fileserver_smb1_done",
+            "maptoguest",
+            "simpleserver",
+            "backupfromdc",
+            "restoredc",
+            "renamedc",
+            "offlinebackupdc",
+            "labdc",
+            "preforkrestartdc",
+            "proclimitdc",
+            "promoted_dc",
+            "vampire_dc",
+            "rodc",
+            "ad_dc_default",
+            "ad_dc_default_smb1",
+            "ad_dc_default_smb1_done",
+            "ad_dc_slowtests",
+            "schema_pair_dc",
+            "schema_dc",
+            "clusteredmember",
+            "ad_dc_fips",
+            "ad_member_fips",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("install", "make install"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "samba-nt4": {
+        "dependency": "samba-nt4-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(include_envs=[
+            "nt4_dc",
+            "nt4_dc_smb1",
+            "nt4_dc_smb1_done",
+            "nt4_dc_schannel",
+            "nt4_member",
+            "simpleserver",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-fileserver": {
+        "dependency": "samba-h5l-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(include_envs=[
+            "fileserver",
+            "fileserver_smb1",
+            "fileserver_smb1_done",
+            "maptoguest",
+            "ktest", # ktest is also tested in samba-ktest-mit samba
+                     # and samba-mitkrb5 but is tested here against
+                     # a system Heimdal
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-fileserver-without-smb1": {
+        "dependency": "samba-without-smb1-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(include_envs=["fileserver"])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    # This is a full build without the AD DC so we test the build with
+    # MIT Kerberos from the current system.  Runtime behaviour is
+    # confirmed via the ktest (static ccache and keytab) environment
+
+    # This environment also used to confirm we can still build with --with-libunwind
+    "samba-ktest-mit": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
+            ("make", "make -j"),
+            ("test", make_test(include_envs=[
+            "ktest", # ktest is also tested in fileserver, samba and
+                     # samba-mitkrb5 but is tested here against a
+                     # system MIT krb5
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-admem": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(include_envs=[
+            "ad_member",
+            "ad_member_idmap_rid",
+            "admem_idmap_autorid",
+            "ad_member_idmap_ad",
+            "ad_member_rfc2307",
+            "ad_member_idmap_nss",
+            "ad_member_offlogon",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-no-opath1": {
+        "dependency": "samba-no-opath-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(
+                cmd="make testonly DISABLE_OPATH=1",
+                include_envs=[
+                "nt4_dc",
+                "nt4_dc_smb1",
+                "nt4_dc_smb1_done",
+                "nt4_dc_schannel",
+                "nt4_member",
+                "simpleserver",
+                ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", "script/clean-source-tree.sh"),
+        ],
+    },
+
+    "samba-no-opath2": {
+        "dependency": "samba-no-opath-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(
+                cmd="make testonly DISABLE_OPATH=1",
+                include_envs=[
+                "fileserver",
+                "fileserver_smb1",
+                "fileserver_smb1_done",
+                ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", "script/clean-source-tree.sh"),
+        ],
+    },
+
+    "samba-ad-dc-1": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "ad_dc",
+            "ad_dc_smb1",
+            "ad_dc_smb1_done",
+            "ad_dc_no_nss",
+            "ad_dc_no_ntlm",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-ad-dc-2": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "vampire_dc",
+            "vampire_2000_dc",
+            "rodc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-ad-dc-3": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "promoted_dc",
+            "chgdcpass",
+            "preforkrestartdc",
+            "proclimitdc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-ad-dc-4a": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "fl2000dc",
+            "ad_member_oneway",
+            "fl2003dc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+    "samba-ad-dc-4b": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "fl2008dc",
+            "fl2008r2dc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-ad-dc-5": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-ad-dc-6": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-schemaupgrade": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
+    # This is currently the longest task, so we don't randomly delay it.
+    "samba-ad-dc-ntvfs": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=["ad_dc_ntvfs"])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    # Test fips compliance
+    "samba-fips": {
+        "dependency": "samba-mit-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
+            # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    # run the backup/restore testenvs separately as they're fairly standalone
+    # (and CI seems to max out at ~3 different DCs running at once)
+    "samba-ad-back1": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(include_envs=[
+            "backupfromdc",
+            "restoredc",
+            "renamedc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+    "samba-ad-back2": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(include_envs=[
+            "backupfromdc",
+            "offlinebackupdc",
+            "labdc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-admem-mit": {
+        "dependency": "samba-mit-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "ad_member",
+            "ad_member_idmap_rid",
+            "admem_idmap_autorid",
+            "ad_member_idmap_ad",
+            "ad_member_rfc2307",
+            "ad_member_idmap_nss",
+            "ad_member_offlogon",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-addc-mit-1": {
+        "dependency": "samba-mit-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "ad_dc",
+            "ad_dc_smb1",
+            "ad_dc_smb1_done",
+            "ad_dc_no_nss",
+            "ad_dc_no_ntlm",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-addc-mit-4a": {
+        "dependency": "samba-mit-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "fl2000dc",
+            "ad_member_oneway",
+            "fl2003dc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+    "samba-addc-mit-4b": {
+        "dependency": "samba-mit-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "fl2008dc",
+            "fl2008r2dc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-test-only": {
+        "sequence": [
+            ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
+            ("make", "make -j"),
+            ("test", make_test(TESTS="${TESTS}")),
+            ("lcov", LCOV_CMD),
+        ],
+    },
+
+    # Test cross-compile infrastructure
+    "samba-xc": {
+        "sequence": [
+            ("random-sleep", random_sleep(900, 1500)),
+            ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
+            ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
+            " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
+            ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
+            ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
+            " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
+            ("compare-results", "script/compare_cc_results.py "
+            "./bin/c4che/default{} "
+            "./bin-xe/c4che/default{} "
+            "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
+            ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
+            ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
+            " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
+            ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
+            " = \"'1234'\"".format(CACHE_SUFFIX)),
+            ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
+            ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
+            " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
+            " ; test $? -ne 0"),
+        ],
+    },
+
+    # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
+    "samba-o3": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
+            ("make", "make -j"),
+            ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
+            ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
+            ("lcov", LCOV_CMD),
+            ("install", "make install"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "samba-32bit": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("configure", "./configure.developer --abi-check-disable --disable-warnings-as-errors" + samba_configure_params),
+            ("make", "make -j"),
+            ("nonetest", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
+            ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
+            ("ktest", make_test(cmd='make test', include_envs=["ktest"])),
+            ("install", "make install"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "samba-ctdb": {
+        "sequence": [
+            ("random-sleep", random_sleep(900, 1500)),
+
+        # make sure we have tdb around:
+            ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
+            ("tdb-make", "cd lib/tdb && make"),
+            ("tdb-install", "cd lib/tdb && make install"),
+
+        # build samba with cluster support (also building ctdb):
+            ("samba-configure",
+         "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
+         "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
+         "./configure.developer ${PREFIX} "
+         "--with-selftest-prefix=./bin/ab "
+         "--with-cluster-support "
+         "--without-ad-dc "
+         "--bundled-libraries=!tdb"),
+            ("samba-make", "make"),
+            ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
+            ("samba-install", "make install"),
+            ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
+
+            ("test", make_test(
+                cmd='PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH make test',
+                INJECT_SELFTEST_PREFIX=0,
+                include_envs=["clusteredmember"])
+            ),
+
+        # clean up:
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+            ("ctdb-clean", "cd ./ctdb && make clean"),
+        ],
+    },
+
+    "samba-libs": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
+            ("talloc-make", "cd lib/talloc && make"),
+            ("talloc-install", "cd lib/talloc && make install"),
+
+            ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
+            ("tdb-make", "cd lib/tdb && make"),
+            ("tdb-install", "cd lib/tdb && make install"),
+
+            ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
+            ("tevent-make", "cd lib/tevent && make"),
+            ("tevent-install", "cd lib/tevent && make install"),
+
+            ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
+            ("ldb-make", "cd lib/ldb && make"),
+            ("ldb-install", "cd lib/ldb && make install"),
+
+            ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"),
+            ("nondevel-make", "make -j"),
+            ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
+            ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
+            ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
+            ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
+            ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
+            ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
+            ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
+            ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
+            ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
+            ("nondevel-no-public-nss_winbind",
+                check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
+            ("nondevel-no-public-nss_wins",
+                check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
+            ("nondevel-no-public-libwbclient",
+                check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
+            ("nondevel-no-public-pam_winbind",
+                check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
+            ("nondevel-no-public-winbind_krb5_locator",
+                check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
+            ("nondevel-no-public-async_dns_krb5_locator",
+                check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
+            ("nondevel-install", "make -j install"),
+            ("nondevel-dist", "make dist"),
+
+            ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
+            ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
+            ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
+            ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
+            ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
+            ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
+            ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
+            ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
+            ("prefix-no-public-nss_winbind",
+                check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
+            ("prefix-no-public-nss_wins",
+                check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
+            ("prefix-no-public-libwbclient",
+                check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
+            ("prefix-no-public-pam_winbind",
+                check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
+            ("prefix-no-public-winbind_krb5_locator",
+                check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
+                              "service_locator")),
+            ("prefix-no-public-async_dns_krb5_locator",
+                check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
+                              "service_locator")),
+
+            # retry with all modules shared
+            ("allshared-distclean", "make distclean"),
+            ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
+            ("allshared-make", "make -j"),
+            ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
+            ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
+            ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
+            ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
+            ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
+            ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
+            ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
+            ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
+            ("allshared-no-public-nss_winbind",
+                check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
+            ("allshared-no-public-nss_wins",
+                check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
+            ("allshared-no-public-libwbclient",
+                check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
+            ("allshared-no-public-pam_winbind",
+                check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
+            ("allshared-no-public-winbind_krb5_locator",
+                check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
+            ("allshared-no-public-async_dns_krb5_locator",
+                check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
+        ],
+    },
+
+    "samba-fuzz": {
+        "sequence": [
+        # build the fuzzers (static) via the oss-fuzz script
+            ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
+            ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
+        ],
+    },
+
+    # * Test smbd and smbtorture can build semi-static
+    #
+    # * Test Samba without python still builds.
+    #
+    # When this test fails due to more use of Python, the expectations
+    # is that the newly failing part of the code should be disabled
+    # when --disable-python is set (rather than major work being done
+    # to support this environment).
+    #
+    # The target here is for vendors shipping a minimal smbd.
+    "samba-minimal-smbd": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+
+        # build with all modules static
+            ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
+            ("allstatic-make", "make -j"),
+            ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
+            ("allstatic-lcov", LCOV_CMD),
+            ("allstatic-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("allstatic-def-clean", "make clean"),
+
+        # force all libraries as private
+            ("allprivate-def-distclean", "make distclean"),
+            ("allprivate-def-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL"),
+            ("allprivate-def-make", "make -j"),
+            # note wrapper libraries need to be public
+            ("allprivate-def-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so' | wc -l | grep -q '^0'"),
+            ("allprivate-def-only-private-ext", "ls ./bin/shared/private | egrep 'private-samba' | wc -l | grep -q '^0' && exit 1; exit 0"),
+            ("allprivate-def-no-non-private-ext", "ls ./bin/shared/private | egrep -v 'private-samba|^libpypamtest.so$' | wc -l | grep -q '^0'"),
+            ("allprivate-def-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
+            ("allprivate-def-lcov", LCOV_CMD),
+            ("allprivate-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("allprivate-def-clean", "make clean"),
+
+        # force all libraries as private with a non default
+        # extension and 2 exceptions
+            ("allprivate-ext-distclean", "make distclean"),
+            ("allprivate-ext-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL --private-library-extension=private-library --private-extension-exception=pac,ndr"),
+            ("allprivate-ext-make", "make -j"),
+            # note wrapper libraries need to be public
+            ("allprivate-ext-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so' | wc -l | grep -q '^0'"),
+            ("allprivate-ext-no-private-default-ext", "ls ./bin/shared/private | grep 'private-samba' | wc -l | grep -q '^0'"),
+            ("allprivate-ext-has-private-ext", "ls ./bin/shared/private | grep 'private-library' | wc -l | grep -q '^0' && exit 1; exit 0"),
+            ("allprivate-ext-libndr-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libndr' | wc -l | grep -q '^1'"),
+            ("allprivate-ext-libpac-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libpac' | wc -l | grep -q '^1'"),
+            ("allprivate-ext-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
+            ("allprivate-ext-lcov", LCOV_CMD),
+            ("allprivate-ext-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("allprivate-ext-clean", "make clean"),
+
+        # retry with nonshared smbd and smbtorture
+            ("nonshared-distclean", "make distclean"),
+            ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
+            ("nonshared-make", "make -j"),
+            ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
+            ("nonshared-lcov", LCOV_CMD),
+            ("nonshared-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("nonshared-clean", "make clean"),
+        ],
+    },
+
+    "samba-nopython": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+
+            ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
+            ("make", "make -j"),
+            ("find-python", "script/find_python.sh ${PREFIX}"),
+            ("test", "make test-nopython"),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+
+            ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
+            ("talloc-make", "cd lib/talloc && make"),
+            ("talloc-install", "cd lib/talloc && make install"),
+
+            ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
+            ("tdb-make", "cd lib/tdb && make"),
+            ("tdb-install", "cd lib/tdb && make install"),
+
+            ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
+            ("tevent-make", "cd lib/tevent && make"),
+            ("tevent-install", "cd lib/tevent && make install"),
+
+            ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
+            ("ldb-make", "cd lib/ldb && make"),
+            ("ldb-install", "cd lib/ldb && make install"),
+
+        # retry against installed library packages, but no required modules
+            ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
+            ("libs-make", "make -j"),
+            ("libs-install", "make install"),
+            ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("libs-clean", "make clean"),
+
+        ],
+    },
+
+    "samba-codecheck": {
+        "sequence": [
+            ("run", "script/check-shell-scripts.sh ."),
+            ("run", "script/codespell.sh ."),
+        ],
+    },
+
+    "ldb": {
+        "sequence": [
+            ("random-sleep", random_sleep(60, 600)),
+            ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+            ("make", "make"),
+            ("install", "make install"),
+            ("test", "make test"),
+            ("lcov", LCOV_CMD),
+            ("clean", "make clean"),
+            ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
+            ("make-no-lmdb", "make"),
+            ("test-no-lmdb", "make test"),
+            ("lcov-no-lmdb", LCOV_CMD),
+            ("install-no-lmdb", "make install"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("distcheck", "make distcheck"),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "tdb": {
+        "sequence": [
+            ("random-sleep", random_sleep(60, 600)),
+            ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+            ("make", "make"),
+            ("install", "make install"),
+            ("test", "make test"),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("distcheck", "make distcheck"),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "talloc": {
+        "sequence": [
+            ("random-sleep", random_sleep(60, 600)),
+            ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+            ("make", "make"),
+            ("install", "make install"),
+            ("test", "make test"),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("distcheck", "make distcheck"),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "replace": {
+        "sequence": [
+            ("random-sleep", random_sleep(60, 600)),
+            ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+            ("make", "make"),
+            ("install", "make install"),
+            ("test", "make test"),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("distcheck", "make distcheck"),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "tevent": {
+        "sequence": [
+            ("random-sleep", random_sleep(60, 600)),
+            ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+            ("make", "make"),
+            ("install", "make install"),
+            ("test", "make test"),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("distcheck", "make distcheck"),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "pidl": {
+        "git-clone-required": True,
+        "sequence": [
+            ("random-sleep", random_sleep(60, 600)),
+            ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
+            ("touch", "touch *.yp"),
+            ("make", "make"),
+            ("test", "make test"),
+            ("install", "make install"),
+            ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+        ],
+    },
+
+    # these are useful for debugging autobuild
+    "pass": {
+        "sequence": [
+            ("pass", 'echo passing && /bin/true'),
+        ],
+    },
+    "fail": {
+        "sequence": [
+            ("fail", 'echo failing && /bin/false'),
+        ],
+    },
+}
+
+defaulttasks = list(tasks.keys())
+
+defaulttasks.remove("pass")
+defaulttasks.remove("fail")
+
+# The build tasks will be brought in by the test tasks as needed
+defaulttasks.remove("samba-def-build")
+defaulttasks.remove("samba-nt4-build")
+defaulttasks.remove("samba-mit-build")
+defaulttasks.remove("samba-h5l-build")
+defaulttasks.remove("samba-no-opath-build")
+
+# This is not a normal test, but a task to support manually running
+# one test under autobuild
+defaulttasks.remove("samba-test-only")
+
+# Only built on GitLab CI and not in the default autobuild because it
+# uses too much space (4GB of semi-static binaries)
+defaulttasks.remove("samba-fuzz")
+
+# The FIPS build runs only in GitLab CI on a current Fedora Docker
+# container where a simulated FIPS mode is possible.
+defaulttasks.remove("samba-fips")
+
+# The MIT build runs on a current Fedora where an up to date MIT KDC
+# is already packaged.  This avoids needing to backport a current MIT
+# to the default Ubuntu 18.04, particularly during development, and
+# the need to install on the shared sn-devel-184.
+
+defaulttasks.remove("samba-mitkrb5")
+defaulttasks.remove("samba-admem-mit")
+defaulttasks.remove("samba-addc-mit-1")
+defaulttasks.remove("samba-addc-mit-4a")
+defaulttasks.remove("samba-addc-mit-4b")
+
+defaulttasks.remove("samba-32bit")
+
+if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
+    defaulttasks.remove("samba-o3")
+
+
+def do_print(msg):
+    logger.info(msg)
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+def do_debug(msg):
+    logger.debug(msg)
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+
+def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
+    if show is None:
+        do_debug("Running: '%s' in '%s'" % (cmd, dir))
+    elif show:
+        do_print("Running: '%s' in '%s'" % (cmd, dir))
+
+    if output:
+        out = check_output([cmd], shell=True, cwd=dir)
+        return out.decode(encoding='utf-8', errors='backslashreplace')
+    elif checkfail:
+        return check_call(cmd, shell=True, cwd=dir)
+    else:
+        return call(cmd, shell=True, cwd=dir)
+
+def rmdir_force(dirname, re_raise=True):
+    try:
+        run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
+                dirname, dirname, dirname), output=True, show=True)
+    except CalledProcessError as e:
+        do_print("Failed: '%s'" % (str(e)))
+        run_cmd("tree %s" % dirname, output=True, show=True)
+        if re_raise:
+            raise
+        return False
+    return True
+
+class builder(object):
+    '''handle build of one directory'''
+
+    def __init__(self, name, definition):
+        self.name = name
+        self.dir = builddirs.get(name, '.')
+        self.tag = self.name.replace('/', '_')
+        self.definition = definition
+        self.sequence = definition["sequence"]
+        self.git_clone_required = False
+        if "git-clone-required" in definition:
+            self.git_clone_required = bool(definition["git-clone-required"])
+        self.proc = None
+        self.done = False
+        self.next = 0
+        self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
+        self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
+        do_debug("stdout for %s in %s" % (self.name, self.stdout_path))
+        do_debug("stderr for %s in %s" % (self.name, self.stderr_path))
+        run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
+        self.stdout = open(self.stdout_path, 'w')
+        self.stderr = open(self.stderr_path, 'w')
+        self.stdin  = open("/dev/null", 'r')
+        self.builder_dir = "%s/%s" % (testbase, self.tag)
+        self.test_source_dir = self.builder_dir
+        self.cwd = "%s/%s" % (self.builder_dir, self.dir)
+        self.selftest_prefix = "%s/bin/ab" % (self.cwd)
+        self.prefix = "%s/%s" % (test_prefix, self.tag)
+        self.consumers = []
+        self.producer = None
+
+        if self.git_clone_required:
+            assert "dependency" not in definition
+
+    def mark_existing(self):
+        do_debug('%s: Mark as existing dependency' % self.name)
+        self.next = len(self.sequence)
+        self.done = True
+
+    def add_consumer(self, consumer):
+        do_debug("%s: add consumer: %s" % (self.name, consumer.name))
+        consumer.producer = self
+        consumer.test_source_dir = self.test_source_dir
+        self.consumers.append(consumer)
+
+    def start_next(self):
+        if self.producer is not None:
+            if not self.producer.done:
+                do_debug("%s: Waiting for producer: %s" % (self.name, self.producer.name))
+                return
+
+        if self.next == 0:
+            rmdir_force(self.builder_dir)
+            rmdir_force(self.prefix)
+            if self.producer is not None:
+                run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
+            elif not self.git_clone_required:
+                run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
+            else:
+                run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
+
+        if self.next == len(self.sequence):
+            if not self.done:
+                do_print('%s: Completed OK' % self.name)
+                self.done = True
+            if not options.nocleanup and len(self.consumers) == 0:
+                do_print('%s: Cleaning up' % self.name)
+                rmdir_force(self.builder_dir)
+                rmdir_force(self.prefix)
+            for consumer in self.consumers:
+                if consumer.next != 0:
+                    continue
+                do_print('%s: Starting consumer %s' % (self.name, consumer.name))
+                consumer.start_next()
+            if self.producer is not None:
+                self.producer.consumers.remove(self)
+                assert self.producer.done
+                self.producer.start_next()
+            do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
+            return
+        (self.stage, self.cmd) = self.sequence[self.next]
+        self.cmd = self.cmd.replace("${PYTHON_PREFIX}",
+                                    get_path(name='platlib',
+                                             scheme="posix_prefix",
+                                             vars={"base": self.prefix,
+                                                   "platbase": self.prefix}))
+        self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
+        self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
+        self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
+        self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
+        self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
+        self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
+        self.cmd = self.cmd.replace("${NAME}", self.name)
+        self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
+        do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
+        self.proc = Popen(self.cmd, shell=True,
+                          close_fds=True, cwd=self.cwd,
+                          stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
+        self.next += 1
+
+def expand_dependencies(n):
+    deps = list()
+    if "dependency" in tasks[n]:
+        depname = tasks[n]["dependency"]
+        assert depname in tasks
+        sdeps = expand_dependencies(depname)
+        assert n not in sdeps
+        for sdep in sdeps:
+            deps.append(sdep)
+        deps.append(depname)
+    return deps
+
+
+class buildlist(object):
+    '''handle build of multiple directories'''
+
+    def __init__(self, tasknames, rebase_url, rebase_branch="master"):
+        self.tail_proc = None
+        self.retry = None
+        if not tasknames:
+            if options.restrict_tests:
+                tasknames = ["samba-test-only"]
+            else:
+                tasknames = defaulttasks
+
+        given_tasknames = tasknames.copy()
+        implicit_tasknames = []
+        for n in given_tasknames:
+            deps = expand_dependencies(n)
+            for dep in deps:
+                if dep in given_tasknames:
+                    continue
+                if dep in implicit_tasknames:
+                    continue
+                implicit_tasknames.append(dep)
+
+        tasknames = implicit_tasknames.copy()
+        tasknames.extend(given_tasknames)
+        do_debug("given_tasknames: %s" % given_tasknames)
+        do_debug("implicit_tasknames: %s" % implicit_tasknames)
+        do_debug("tasknames: %s" % tasknames)
+        self.tlist = [builder(n, tasks[n]) for n in tasknames]
+
+        if options.retry:
+            rebase_remote = "rebaseon"
+            retry_task = {
+                    "git-clone-required": True,
+                    "sequence": [
+                            ("retry",
+                            '''set -e
+                            git remote add -t %s %s %s
+                            git fetch %s
+                            while :; do
+                              sleep 60
+                              git describe %s/%s > old_remote_branch.desc
+                              git fetch %s
+                              git describe %s/%s > remote_branch.desc
+                              diff old_remote_branch.desc remote_branch.desc
+                            done
+                           ''' % (
+                               rebase_branch, rebase_remote, rebase_url,
+                               rebase_remote,
+                               rebase_remote, rebase_branch,
+                               rebase_remote,
+                               rebase_remote, rebase_branch
+                            ))]}
+
+            self.retry = builder('retry', retry_task)
+            self.need_retry = False
+
+        if options.skip_dependencies:
+            for b in self.tlist:
+                if b.name in implicit_tasknames:
+                    b.mark_existing()
+
+        for b in self.tlist:
+            do_debug("b.name=%s" % b.name)
+            if "dependency" not in b.definition:
+                continue
+            depname = b.definition["dependency"]
+            do_debug("b.name=%s: dependency:%s" % (b.name, depname))
+            for p in self.tlist:
+                if p.name == depname:
+                    p.add_consumer(b)
+
+    def kill_kids(self):
+        if self.tail_proc is not None:
+            self.tail_proc.terminate()
+            self.tail_proc.wait()
+            self.tail_proc = None
+        if self.retry is not None:
+            self.retry.proc.terminate()
+            self.retry.proc.wait()
+            self.retry = None
+        for b in self.tlist:
+            if b.proc is not None:
+                run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
+                b.proc.terminate()
+                b.proc.wait()
+                b.proc = None
+
+    def wait_one(self):
+        while True:
+            none_running = True
+            for b in self.tlist:
+                if b.proc is None:
+                    continue
+                none_running = False
+                b.status = b.proc.poll()
+                if b.status is None:
+                    continue
+                b.proc = None
+                return b
+            if options.retry:
+                ret = self.retry.proc.poll()
+                if ret is not None:
+                    self.need_retry = True
+                    self.retry = None
+                    return None
+            if none_running:
+                return None
+            time.sleep(0.1)
+
+    def run(self):
+        for b in self.tlist:
+            b.start_next()
+        if options.retry:
+            self.retry.start_next()
+        while True:
+            b = self.wait_one()
+            if options.retry and self.need_retry:
+                self.kill_kids()
+                do_print("retry needed")
+                return (0, None, None, None, "retry")
+            if b is None:
+                break
+            if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
+                self.kill_kids()
+                return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
+            b.start_next()
+        self.kill_kids()
+        return (0, None, None, None, "All OK")
+
+    def write_system_info(self, filename):
+        with open(filename, 'w') as f:
+            for cmd in ['uname -a',
+                        'lsb_release -a',
+                        'free',
+                        'mount',
+                        'cat /proc/cpuinfo',
+                        'cc --version',
+                        'df -m .',
+                        'df -m %s' % testbase]:
+                try:
+                    out = run_cmd(cmd, output=True, checkfail=False)
+                except CalledProcessError as e:
+                    out = "<failed: %s>" % str(e)
+                print('### %s' % cmd, file=f)
+                print(out, file=f)
+                print(file=f)
+
+    def tarlogs(self, fname):
+        with tarfile.open(fname, "w:gz") as tar:
+            for b in self.tlist:
+                tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
+                tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
+            if os.path.exists("autobuild.log"):
+                tar.add("autobuild.log")
+            filename = 'system-info.txt'
+            self.write_system_info(filename)
+            tar.add(filename)
+
+    def remove_logs(self):
+        for b in self.tlist:
+            os.unlink(b.stdout_path)
+            os.unlink(b.stderr_path)
+
+    def start_tail(self):
+        cmd = ["tail", "-f"]
+        for b in self.tlist:
+            cmd.append(b.stdout_path)
+            cmd.append(b.stderr_path)
+        self.tail_proc = Popen(cmd, close_fds=True)
+
+
+def cleanup(do_raise=False):
+    if options.nocleanup:
+        return
+    run_cmd("stat %s || true" % test_tmpdir, show=True)
+    run_cmd("stat %s" % testbase, show=True)
+    do_print("Cleaning up %r" % cleanup_list)
+    for d in cleanup_list:
+        ok = rmdir_force(d, re_raise=False)
+        if ok:
+            continue
+        if os.path.isdir(d):
+            do_print("Killing, waiting and retry")
+            run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
+        else:
+            do_print("Waiting and retry")
+        time.sleep(1)
+        rmdir_force(d, re_raise=do_raise)
+
+
+def daemonize(logfile):
+    pid = os.fork()
+    if pid == 0:  # Parent
+        os.setsid()
+        pid = os.fork()
+        if pid != 0:  # Actual daemon
+            os._exit(0)
+    else:  # Grandparent
+        os._exit(0)
+
+    import resource      # Resource usage information.
+    maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+    if maxfd == resource.RLIM_INFINITY:
+        maxfd = 1024  # Rough guess at maximum number of open file descriptors.
+    for fd in range(0, maxfd):
+        try:
+            os.close(fd)
+        except OSError:
+            pass
+    os.open(logfile, os.O_RDWR | os.O_CREAT)
+    os.dup2(0, 1)
+    os.dup2(0, 2)
+
+
+def write_pidfile(fname):
+    '''write a pid file, cleanup on exit'''
+    with open(fname, mode='w') as f:
+        f.write("%u\n" % os.getpid())
+
+
+def rebase_tree(rebase_url, rebase_branch="master"):
+    rebase_remote = "rebaseon"
+    do_print("Rebasing on %s" % rebase_url)
+    run_cmd("git describe HEAD", show=True, dir=test_master)
+    run_cmd("git remote add -t %s %s %s" %
+            (rebase_branch, rebase_remote, rebase_url),
+            show=True, dir=test_master)
+    run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
+    if options.fix_whitespace:
+        run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
+                (rebase_remote, rebase_branch),
+                show=True, dir=test_master)
+    else:
+        run_cmd("git rebase --force-rebase %s/%s" %
+                (rebase_remote, rebase_branch),
+                show=True, dir=test_master)
+    diff = run_cmd("git --no-pager diff HEAD %s/%s" %
+                   (rebase_remote, rebase_branch),
+                   dir=test_master, output=True)
+    if diff == '':
+        do_print("No differences between HEAD and %s/%s - exiting" %
+                 (rebase_remote, rebase_branch))
+        sys.exit(0)
+    run_cmd("git describe %s/%s" %
+            (rebase_remote, rebase_branch),
+            show=True, dir=test_master)
+    run_cmd("git describe HEAD", show=True, dir=test_master)
+    run_cmd("git --no-pager diff --stat HEAD %s/%s" %
+            (rebase_remote, rebase_branch),
+            show=True, dir=test_master)
+
+
+def push_to(push_url, push_branch="master"):
+    push_remote = "pushto"
+    do_print("Pushing to %s" % push_url)
+    if options.mark:
+        run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
+        run_cmd("git commit --amend -c HEAD", dir=test_master)
+        # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
+        # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
+    run_cmd("git remote add -t %s %s %s" %
+            (push_branch, push_remote, push_url),
+            show=True, dir=test_master)
+    run_cmd("git push %s +HEAD:%s" %
+            (push_remote, push_branch),
+            show=True, dir=test_master)
+
+
+def send_email(subject, text, log_tar):
+    if options.email is None:
+        do_print("not sending email because the recipient is not set")
+        do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
+                 (subject, text))
+        return
+    outer = MIMEMultipart()
+    outer['Subject'] = subject
+    outer['To'] = options.email
+    outer['From'] = options.email_from
+    outer['Date'] = email.utils.formatdate(localtime=True)
+    outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
+    outer.attach(MIMEText(text, 'plain', 'utf-8'))
+    if options.attach_logs:
+        with open(log_tar, 'rb') as fp:
+            msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
+        # Set the filename parameter
+        msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
+        outer.attach(msg)
+    content = outer.as_string()
+    s = smtplib.SMTP(options.email_server)
+    email_user = os.getenv('SMTP_USERNAME')
+    email_password = os.getenv('SMTP_PASSWORD')
+    if email_user is not None:
+        s.starttls()
+        s.login(email_user, email_password)
+
+    s.sendmail(options.email_from, [options.email], content)
+    s.set_debuglevel(1)
+    s.quit()
+
+
+def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
+                  elapsed_time, log_base=None, add_log_tail=True):
+    '''send an email to options.email about the failure'''
+    elapsed_minutes = elapsed_time / 60.0
+    if log_base is None:
+        log_base = gitroot
+    text = '''
+Dear Developer,
+
+Your autobuild on %s failed after %.1f minutes
+when trying to test %s with the following error:
+
+   %s
+
+the autobuild has been abandoned. Please fix the error and resubmit.
+
+A summary of the autobuild process is here:
+
+  %s/autobuild.log
+''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
+
+    if options.restrict_tests:
+        text += """
+The build was restricted to tests matching %s\n""" % options.restrict_tests
+
+    if failed_task != 'rebase':
+        text += '''
+You can see logs of the failed task here:
+
+  %s/%s.stdout
+  %s/%s.stderr
+
+or you can get full logs of all tasks in this job here:
+
+  %s/logs.tar.gz
+
+The top commit for the tree that was built was:
+
+%s
+
+''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
+
+    log_stdout = "%s/%s.stdout" % (gitroot, failed_tag)
+    if add_log_tail and os.access(log_stdout, os.R_OK):
+        f = open(log_stdout, 'r')
+        lines = f.readlines()
+        log_tail = "".join(lines[-50:])
+        num_lines = len(lines)
+        log_stderr = "%s/%s.stderr" % (gitroot, failed_tag)
+        if num_lines < 50 and os.access(log_stderr, os.R_OK):
+            # Also include stderr (compile failures) if < 50 lines of stdout
+            f = open(log_stderr, 'r')
+            log_tail += "".join(f.readlines()[-(50 - num_lines):])
+
+        text += '''
+The last 50 lines of log messages:
+
+%s
+    ''' % log_tail
+        f.close()
+
+    logs = os.path.join(gitroot, 'logs.tar.gz')
+    send_email('autobuild[%s] failure on %s for task %s during %s'
+               % (options.branch, platform.node(), failed_task, failed_stage),
+               text, logs)
+
+
+def email_success(elapsed_time, log_base=None):
+    '''send an email to options.email about a successful build'''
+    if log_base is None:
+        log_base = gitroot
+    text = '''
+Dear Developer,
+
+Your autobuild on %s has succeeded after %.1f minutes.
+
+''' % (platform.node(), elapsed_time / 60.)
+
+    if options.restrict_tests:
+        text += """
+The build was restricted to tests matching %s\n""" % options.restrict_tests
+
+    if options.keeplogs:
+        text += '''
+
+you can get full logs of all tasks in this job here:
+
+  %s/logs.tar.gz
+
+''' % log_base
+
+    text += '''
+The top commit for the tree that was built was:
+
+%s
+''' % top_commit_msg
+
+    logs = os.path.join(gitroot, 'logs.tar.gz')
+    send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
+               text, logs)
+
+
+# get the top commit message, for emails
+top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
+
+try:
+    if options.skip_dependencies:
+        run_cmd("stat %s" % testbase, dir=testbase, output=True)
+    else:
+        os.makedirs(testbase)
+except Exception as reason:
+    raise Exception("Unable to create %s : %s" % (testbase, reason))
+cleanup_list.append(testbase)
+
+if options.daemon:
+    logfile = os.path.join(testbase, "log")
+    do_print("Forking into the background, writing progress to %s" % logfile)
+    daemonize(logfile)
+
+write_pidfile(gitroot + "/autobuild.pid")
+
+start_time = time.time()
+
+while True:
+    try:
+        run_cmd("rm -rf %s" % test_tmpdir, show=True)
+        os.makedirs(test_tmpdir)
+        # The waf uninstall code removes empty directories all the way
+        # up the tree.  Creating a file in test_tmpdir stops it from
+        # being removed.
+        run_cmd("touch %s" % os.path.join(test_tmpdir,
+                                          ".directory-is-not-empty"), show=True)
+        run_cmd("stat %s" % test_tmpdir, show=True)
+        run_cmd("stat %s" % testbase, show=True)
+        if options.skip_dependencies:
+            run_cmd("stat %s" % test_master, dir=testbase, output=True)
+        else:
+            run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
+    except Exception:
+        cleanup()
+        raise
+
+    try:
+        if options.rebase is not None:
+            rebase_tree(options.rebase, rebase_branch=options.branch)
+    except Exception:
+        cleanup_list.append(gitroot + "/autobuild.pid")
+        cleanup()
+        elapsed_time = time.time() - start_time
+        email_failure(-1, 'rebase', 'rebase', 'rebase',
+                      'rebase on %s failed' % options.branch,
+                      elapsed_time, log_base=options.log_base)
+        sys.exit(1)
+
+    try:
+        blist = buildlist(args, options.rebase, rebase_branch=options.branch)
+        if options.tail:
+            blist.start_tail()
+        (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
+        if status != 0 or errstr != "retry":
+            break
+        cleanup(do_raise=True)
+    except Exception:
+        cleanup()
+        raise
+
+cleanup_list.append(gitroot + "/autobuild.pid")
+
+do_print(errstr)
+
+blist.kill_kids()
+if options.tail:
+    do_print("waiting for tail to flush")
+    time.sleep(1)
+
+elapsed_time = time.time() - start_time
+if status == 0:
+    if options.passcmd is not None:
+        do_print("Running passcmd: %s" % options.passcmd)
+        run_cmd(options.passcmd, dir=test_master)
+    if options.pushto is not None:
+        push_to(options.pushto, push_branch=options.branch)
+    if options.keeplogs or options.attach_logs:
+        blist.tarlogs("logs.tar.gz")
+        do_print("Logs in logs.tar.gz")
+    if options.always_email:
+        email_success(elapsed_time, log_base=options.log_base)
+    blist.remove_logs()
+    cleanup()
+    do_print(errstr)
+    sys.exit(0)
+
+# something failed, gather a tar of the logs
+blist.tarlogs("logs.tar.gz")
+
+if options.email is not None:
+    email_failure(status, failed_task, failed_stage, failed_tag, errstr,
+                  elapsed_time, log_base=options.log_base)
+else:
+    elapsed_minutes = elapsed_time / 60.0
+    print('''
+
+####################################################################
+
+AUTOBUILD FAILURE
+
+Your autobuild[%s] on %s failed after %.1f minutes
+when trying to test %s with the following error:
+
+   %s
+
+the autobuild has been abandoned. Please fix the error and resubmit.
+
+####################################################################
+
+''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
+
+cleanup()
+do_print(errstr)
+do_print("Logs in logs.tar.gz")
+sys.exit(status)