summaryrefslogtreecommitdiffstats
path: root/script/autobuild.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 17:20:00 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 17:20:00 +0000
commit8daa83a594a2e98f39d764422bfbdbc62c9efd44 (patch)
tree4099e8021376c7d8c05bdf8503093d80e9c7bad0 /script/autobuild.py
parentInitial commit. (diff)
downloadsamba-8daa83a594a2e98f39d764422bfbdbc62c9efd44.tar.xz
samba-8daa83a594a2e98f39d764422bfbdbc62c9efd44.zip
Adding upstream version 2:4.20.0+dfsg.upstream/2%4.20.0+dfsg
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'script/autobuild.py')
-rwxr-xr-xscript/autobuild.py1902
1 files changed, 1902 insertions, 0 deletions
diff --git a/script/autobuild.py b/script/autobuild.py
new file mode 100755
index 0000000..ecec352
--- /dev/null
+++ b/script/autobuild.py
@@ -0,0 +1,1902 @@
+#!/usr/bin/env python3
+# run tests on all Samba subprojects and push to a git tree on success
+# Copyright Andrew Tridgell 2010
+# released under GNU GPL v3 or later
+
+from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
+import os
+import tarfile
+import sys
+import time
+import random
+from optparse import OptionParser
+import smtplib
+import email
+from email.mime.text import MIMEText
+from email.mime.base import MIMEBase
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from sysconfig import get_path
+import platform
+
+import logging
+
+try:
+ from waflib.Build import CACHE_SUFFIX
+except ImportError:
+ sys.path.insert(0, "./third_party/waf")
+ from waflib.Build import CACHE_SUFFIX
+
+logging.basicConfig(format='%(asctime)s %(message)s')
+logger = logging.getLogger('autobuild')
+logger.setLevel(logging.INFO)
+
+os.environ["PYTHONUNBUFFERED"] = "1"
+
+# This speeds up testing remarkably.
+os.environ['TDB_NO_FSYNC'] = '1'
+
+# allow autobuild to run within git rebase -i
+if "GIT_DIR" in os.environ:
+ del os.environ["GIT_DIR"]
+if "GIT_WORK_TREE" in os.environ:
+ del os.environ["GIT_WORK_TREE"]
+
+def find_git_root():
+ '''get to the top of the git repo'''
+ p = os.getcwd()
+ while p != '/':
+ if os.path.exists(os.path.join(p, ".git")):
+ return p
+ p = os.path.abspath(os.path.join(p, '..'))
+ return None
+
+
+gitroot = find_git_root()
+if gitroot is None:
+ raise Exception("Failed to find git root")
+
+
+def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
+
+parser = OptionParser()
+parser.add_option("--tail", help="show output while running", default=False, action="store_true")
+parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
+parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
+parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
+parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
+ default=def_testbase)
+parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
+ default=None)
+parser.add_option("--passcmd", help="command to run on success", default=None)
+parser.add_option("--verbose", help="show all commands as they are run",
+ default=False, action="store_true")
+parser.add_option("--rebase", help="rebase on the given tree before testing",
+ default=None, type='str')
+parser.add_option("--pushto", help="push to a git url on success",
+ default=None, type='str')
+parser.add_option("--mark", help="add a Tested-By signoff before pushing",
+ default=False, action="store_true")
+parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
+ default=False, action="store_true")
+parser.add_option("--retry", help="automatically retry if master changes",
+ default=False, action="store_true")
+parser.add_option("--email", help="send email to the given address on failure",
+ type='str', default=None)
+parser.add_option("--email-from", help="send email from the given address",
+ type='str', default="autobuild@samba.org")
+parser.add_option("--email-server", help="send email via the given server",
+ type='str', default='localhost')
+parser.add_option("--always-email", help="always send email, even on success",
+ action="store_true")
+parser.add_option("--daemon", help="daemonize after initial setup",
+ action="store_true")
+parser.add_option("--branch", help="the branch to work on (default=master)",
+ default="master", type='str')
+parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
+ default=gitroot, type='str')
+parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
+ default=False, action="store_true")
+parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
+ default='')
+parser.add_option("--enable-coverage", dest='enable_coverage',
+ action="store_const", const='--enable-coverage', default='',
+ help="Add --enable-coverage option while configure")
+
+(options, args) = parser.parse_args()
+
+if options.retry:
+ if options.rebase is None:
+ raise Exception('You can only use --retry if you also rebase')
+
+if options.verbose:
+ logger.setLevel(logging.DEBUG)
+
+if options.full_testbase is not None:
+ testbase = options.full_testbase
+else:
+ testbase = "%s/b%u" % (options.testbase, os.getpid())
+test_master = "%s/master" % testbase
+test_prefix = "%s/prefix" % testbase
+test_tmpdir = "%s/tmp" % testbase
+os.environ['TMPDIR'] = test_tmpdir
+
+if options.enable_coverage:
+ LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
+else:
+ LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
+
+if options.enable_coverage:
+ PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
+else:
+ PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
+
+CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
+
+
+def check_symbols(sofile, expected_symbols=""):
+ return "objdump --dynamic-syms " + sofile + " | " + \
+ "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
+
+if args:
+ # If we are only running specific test,
+ # do not sleep randomly to wait for it to start
+ def random_sleep(low, high):
+ return 'sleep 1'
+else:
+ def random_sleep(low, high):
+ return 'sleep {}'.format(random.randint(low, high))
+
+cleanup_list = []
+
+builddirs = {
+ "ctdb": "ctdb",
+ "ldb": "lib/ldb",
+ "tdb": "lib/tdb",
+ "talloc": "lib/talloc",
+ "replace": "lib/replace",
+ "tevent": "lib/tevent",
+ "pidl": "pidl",
+ "docs-xml": "docs-xml"
+}
+
+ctdb_configure_params = " --enable-developer ${PREFIX}"
+samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
+
+samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
+samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
+samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
+samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
+samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
+samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
+samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
+
+
+def format_option(name, value=None):
+ """Format option as str list."""
+ if value is None: # boolean option
+ return [name]
+ if not isinstance(value, list): # single value option
+ value = [value]
+ # repeatable option
+ return ['{}={}'.format(name, item) for item in value]
+
+
+def make_test(
+ cmd='make testonly',
+ INJECT_SELFTEST_PREFIX=1,
+ TESTS='',
+ include_envs=None,
+ exclude_envs=None):
+
+ test_options = []
+ if include_envs:
+ test_options = format_option('--include-env', include_envs)
+ if exclude_envs:
+ test_options = format_option('--exclude-env', exclude_envs)
+ if test_options:
+ # join envs options to original test options
+ TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
+
+ _options = []
+
+ # Allow getting a full CI with
+ # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
+
+ FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
+
+ if int(FAIL_IMMEDIATELY):
+ _options.append('FAIL_IMMEDIATELY=1')
+ if TESTS:
+ _options.append("TESTS='{}'".format(TESTS))
+
+ if INJECT_SELFTEST_PREFIX:
+ _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
+ _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
+
+ return ' '.join([cmd] + _options)
+
+
+# When updating this list, also update .gitlab-ci.yml to add the job
+# and to make it a dependency of 'page' for the coverage report.
+
+tasks = {
+ "ctdb": {
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("configure", "./configure " + ctdb_configure_params),
+ ("make", "make all"),
+ ("install", "make install"),
+ ("test", "make autotest"),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("clean", "make clean"),
+ ],
+ },
+ "docs-xml": {
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("autoconf", "autoconf"),
+ ("configure", "./configure"),
+ ("make", "make html htmlman"),
+ ("publish-docs", PUBLISH_DOCS),
+ ("clean", "make clean"),
+ ],
+ },
+
+ "samba-def-build": {
+ "git-clone-required": True,
+ "sequence": [
+ ("configure", "./configure.developer" + samba_configure_params),
+ ("make", "make -j"),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("chmod-R-a-w", "chmod -R a-w ."),
+ ],
+ },
+
+ "samba-mit-build": {
+ "git-clone-required": True,
+ "sequence": [
+ ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
+ ("make", "make -j"),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("chmod-R-a-w", "chmod -R a-w ."),
+ ],
+ },
+
+ "samba-nt4-build": {
+ "git-clone-required": True,
+ "sequence": [
+ ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
+ ("make", "make -j"),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("chmod-R-a-w", "chmod -R a-w ."),
+ ],
+ },
+
+ "samba-h5l-build": {
+ "git-clone-required": True,
+ "sequence": [
+ ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
+ ("make", "make -j"),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("chmod-R-a-w", "chmod -R a-w ."),
+ ],
+ },
+
+ "samba-without-smb1-build": {
+ "git-clone-required": True,
+ "sequence": [
+ ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
+ ("make", "make -j"),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("chmod-R-a-w", "chmod -R a-w ."),
+ ],
+ },
+
+ "samba-no-opath-build": {
+ "git-clone-required": True,
+ "sequence": [
+ ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1 -DDISABLE_VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS=1 -DDISABLE_PROC_FDS=1' ./configure.developer --without-ad-dc " + samba_configure_params),
+ ("make", "make -j"),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("chmod-R-a-w", "chmod -R a-w ."),
+ ],
+ },
+
+ # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
+ "samba": {
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("configure", "./configure.developer" + samba_configure_params),
+ ("make", "make -j"),
+ ("test", make_test(exclude_envs=[
+ "none",
+ "nt4_dc",
+ "nt4_dc_smb1",
+ "nt4_dc_smb1_done",
+ "nt4_dc_schannel",
+ "nt4_member",
+ "ad_dc",
+ "ad_dc_smb1",
+ "ad_dc_smb1_done",
+ "ad_dc_backup",
+ "ad_dc_ntvfs",
+ "ad_dc_default",
+ "ad_dc_default_smb1",
+ "ad_dc_slowtests",
+ "ad_dc_no_nss",
+ "ad_dc_no_ntlm",
+ "fl2003dc",
+ "fl2008dc",
+ "fl2008r2dc",
+ "ad_member",
+ "ad_member_idmap_rid",
+ "admem_idmap_autorid",
+ "ad_member_idmap_ad",
+ "ad_member_rfc2307",
+ "ad_member_idmap_nss",
+ "ad_member_oneway",
+ "chgdcpass",
+ "vampire_2000_dc",
+ "fl2000dc",
+ "fileserver",
+ "fileserver_smb1",
+ "fileserver_smb1_done",
+ "maptoguest",
+ "simpleserver",
+ "backupfromdc",
+ "restoredc",
+ "renamedc",
+ "offlinebackupdc",
+ "labdc",
+ "preforkrestartdc",
+ "proclimitdc",
+ "promoted_dc",
+ "vampire_dc",
+ "rodc",
+ "ad_dc_default",
+ "ad_dc_default_smb1",
+ "ad_dc_default_smb1_done",
+ "ad_dc_slowtests",
+ "schema_pair_dc",
+ "schema_dc",
+ "clusteredmember",
+ "ad_dc_fips",
+ "ad_member_fips",
+ ])),
+ ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
+ ("lcov", LCOV_CMD),
+ ("install", "make install"),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("clean", "make clean"),
+ ],
+ },
+
+ # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
+ "samba-mitkrb5": {
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
+ ("make", "make -j"),
+ ("test", make_test(exclude_envs=[
+ "none",
+ "nt4_dc",
+ "nt4_dc_smb1",
+ "nt4_dc_smb1_done",
+ "nt4_dc_schannel",
+ "nt4_member",
+ "ad_dc",
+ "ad_dc_smb1",
+ "ad_dc_smb1_done",
+ "ad_dc_backup",
+ "ad_dc_ntvfs",
+ "ad_dc_default",
+ "ad_dc_default_smb1",
+ "ad_dc_default_smb1_done",
+ "ad_dc_slowtests",
+ "ad_dc_no_nss",
+ "ad_dc_no_ntlm",
+ "fl2003dc",
+ "fl2008dc",
+ "fl2008r2dc",
+ "ad_member",
+ "ad_member_idmap_rid",
+ "admem_idmap_autorid",
+ "ad_member_idmap_ad",
+ "ad_member_rfc2307",
+ "ad_member_idmap_nss",
+ "ad_member_oneway",
+ "chgdcpass",
+ "vampire_2000_dc",
+ "fl2000dc",
+ "fileserver",
+ "fileserver_smb1",
+ "fileserver_smb1_done",
+ "maptoguest",
+ "simpleserver",
+ "backupfromdc",
+ "restoredc",
+ "renamedc",
+ "offlinebackupdc",
+ "labdc",
+ "preforkrestartdc",
+ "proclimitdc",
+ "promoted_dc",
+ "vampire_dc",
+ "rodc",
+ "ad_dc_default",
+ "ad_dc_default_smb1",
+ "ad_dc_default_smb1_done",
+ "ad_dc_slowtests",
+ "schema_pair_dc",
+ "schema_dc",
+ "clusteredmember",
+ "ad_dc_fips",
+ "ad_member_fips",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("install", "make install"),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("clean", "make clean"),
+ ],
+ },
+
+ "samba-nt4": {
+ "dependency": "samba-nt4-build",
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("test", make_test(include_envs=[
+ "nt4_dc",
+ "nt4_dc_smb1",
+ "nt4_dc_smb1_done",
+ "nt4_dc_schannel",
+ "nt4_member",
+ "simpleserver",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-fileserver": {
+ "dependency": "samba-h5l-build",
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("test", make_test(include_envs=[
+ "fileserver",
+ "fileserver_smb1",
+ "fileserver_smb1_done",
+ "maptoguest",
+ "ktest", # ktest is also tested in samba-ktest-mit samba
+ # and samba-mitkrb5 but is tested here against
+ # a system Heimdal
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-fileserver-without-smb1": {
+ "dependency": "samba-without-smb1-build",
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("test", make_test(include_envs=["fileserver"])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ # This is a full build without the AD DC so we test the build with
+ # MIT Kerberos from the current system. Runtime behaviour is
+ # confirmed via the ktest (static ccache and keytab) environment
+
+ # This environment also used to confirm we can still build with --with-libunwind
+ "samba-ktest-mit": {
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
+ ("make", "make -j"),
+ ("test", make_test(include_envs=[
+ "ktest", # ktest is also tested in fileserver, samba and
+ # samba-mitkrb5 but is tested here against a
+ # system MIT krb5
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-admem": {
+ "dependency": "samba-def-build",
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("test", make_test(include_envs=[
+ "ad_member",
+ "ad_member_idmap_rid",
+ "admem_idmap_autorid",
+ "ad_member_idmap_ad",
+ "ad_member_rfc2307",
+ "ad_member_idmap_nss",
+ "ad_member_offlogon",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-no-opath1": {
+ "dependency": "samba-no-opath-build",
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("test", make_test(
+ cmd="make testonly DISABLE_OPATH=1",
+ include_envs=[
+ "nt4_dc",
+ "nt4_dc_smb1",
+ "nt4_dc_smb1_done",
+ "nt4_dc_schannel",
+ "nt4_member",
+ "simpleserver",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", "script/clean-source-tree.sh"),
+ ],
+ },
+
+ "samba-no-opath2": {
+ "dependency": "samba-no-opath-build",
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("test", make_test(
+ cmd="make testonly DISABLE_OPATH=1",
+ include_envs=[
+ "fileserver",
+ "fileserver_smb1",
+ "fileserver_smb1_done",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", "script/clean-source-tree.sh"),
+ ],
+ },
+
+ "samba-ad-dc-1": {
+ "dependency": "samba-def-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=[
+ "ad_dc",
+ "ad_dc_smb1",
+ "ad_dc_smb1_done",
+ "ad_dc_no_nss",
+ "ad_dc_no_ntlm",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-ad-dc-2": {
+ "dependency": "samba-def-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=[
+ "vampire_dc",
+ "vampire_2000_dc",
+ "rodc",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-ad-dc-3": {
+ "dependency": "samba-def-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=[
+ "promoted_dc",
+ "chgdcpass",
+ "preforkrestartdc",
+ "proclimitdc",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-ad-dc-4a": {
+ "dependency": "samba-def-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=[
+ "fl2000dc",
+ "ad_member_oneway",
+ "fl2003dc",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+ "samba-ad-dc-4b": {
+ "dependency": "samba-def-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=[
+ "fl2008dc",
+ "fl2008r2dc",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-ad-dc-5": {
+ "dependency": "samba-def-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=[
+ "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-ad-dc-6": {
+ "dependency": "samba-def-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-schemaupgrade": {
+ "dependency": "samba-def-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
+ # This is currently the longest task, so we don't randomly delay it.
+ "samba-ad-dc-ntvfs": {
+ "dependency": "samba-def-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=["ad_dc_ntvfs"])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ # Test fips compliance
+ "samba-fips": {
+ "dependency": "samba-mit-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
+ # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ # run the backup/restore testenvs separately as they're fairly standalone
+ # (and CI seems to max out at ~3 different DCs running at once)
+ "samba-ad-back1": {
+ "dependency": "samba-def-build",
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("test", make_test(include_envs=[
+ "backupfromdc",
+ "restoredc",
+ "renamedc",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+ "samba-ad-back2": {
+ "dependency": "samba-def-build",
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("test", make_test(include_envs=[
+ "backupfromdc",
+ "offlinebackupdc",
+ "labdc",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-admem-mit": {
+ "dependency": "samba-mit-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=[
+ "ad_member",
+ "ad_member_idmap_rid",
+ "admem_idmap_autorid",
+ "ad_member_idmap_ad",
+ "ad_member_rfc2307",
+ "ad_member_idmap_nss",
+ "ad_member_offlogon",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-addc-mit-1": {
+ "dependency": "samba-mit-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=[
+ "ad_dc",
+ "ad_dc_smb1",
+ "ad_dc_smb1_done",
+ "ad_dc_no_nss",
+ "ad_dc_no_ntlm",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-addc-mit-4a": {
+ "dependency": "samba-mit-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=[
+ "fl2000dc",
+ "ad_member_oneway",
+ "fl2003dc",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+ "samba-addc-mit-4b": {
+ "dependency": "samba-mit-build",
+ "sequence": [
+ ("random-sleep", random_sleep(1, 1)),
+ ("test", make_test(include_envs=[
+ "fl2008dc",
+ "fl2008r2dc",
+ ])),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ],
+ },
+
+ "samba-test-only": {
+ "sequence": [
+ ("configure", "./configure.developer --abi-check-disable" + samba_configure_params),
+ ("make", "make -j"),
+ ("test", make_test(TESTS="${TESTS}")),
+ ("lcov", LCOV_CMD),
+ ],
+ },
+
+ # Test cross-compile infrastructure
+ "samba-xc": {
+ "sequence": [
+ ("random-sleep", random_sleep(900, 1500)),
+ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
+ ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
+ " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
+ ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
+ ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
+ " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
+ ("compare-results", "script/compare_cc_results.py "
+ "./bin/c4che/default{} "
+ "./bin-xe/c4che/default{} "
+ "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
+ ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
+ ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
+ " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
+ ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
+ " = \"'1234'\"".format(CACHE_SUFFIX)),
+ ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
+ ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
+ " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
+ " ; test $? -ne 0"),
+ ],
+ },
+
+ # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
+ "samba-o3": {
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
+ ("make", "make -j"),
+ ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
+ ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
+ ("lcov", LCOV_CMD),
+ ("install", "make install"),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("clean", "make clean"),
+ ],
+ },
+
+ "samba-32bit": {
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("configure", "./configure.developer --abi-check-disable --disable-warnings-as-errors" + samba_configure_params),
+ ("make", "make -j"),
+ ("nonetest", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
+ ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
+ ("ktest", make_test(cmd='make test', include_envs=["ktest"])),
+ ("install", "make install"),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("clean", "make clean"),
+ ],
+ },
+
+ "samba-ctdb": {
+ "sequence": [
+ ("random-sleep", random_sleep(900, 1500)),
+
+ # make sure we have tdb around:
+ ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
+ ("tdb-make", "cd lib/tdb && make"),
+ ("tdb-install", "cd lib/tdb && make install"),
+
+ # build samba with cluster support (also building ctdb):
+ ("samba-configure",
+ "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
+ "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
+ "./configure.developer ${PREFIX} "
+ "--with-selftest-prefix=./bin/ab "
+ "--with-cluster-support "
+ "--without-ad-dc "
+ "--bundled-libraries=!tdb"),
+ ("samba-make", "make"),
+ ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
+ ("samba-install", "make install"),
+ ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
+
+ ("test", make_test(
+ cmd='PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH make test',
+ INJECT_SELFTEST_PREFIX=0,
+ include_envs=["clusteredmember"])
+ ),
+
+ # clean up:
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("clean", "make clean"),
+ ("ctdb-clean", "cd ./ctdb && make clean"),
+ ],
+ },
+
+ "samba-libs": {
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+ ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
+ ("talloc-make", "cd lib/talloc && make"),
+ ("talloc-install", "cd lib/talloc && make install"),
+
+ ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
+ ("tdb-make", "cd lib/tdb && make"),
+ ("tdb-install", "cd lib/tdb && make install"),
+
+ ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
+ ("tevent-make", "cd lib/tevent && make"),
+ ("tevent-install", "cd lib/tevent && make install"),
+
+ ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
+ ("ldb-make", "cd lib/ldb && make"),
+ ("ldb-install", "cd lib/ldb && make install"),
+
+ ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"),
+ ("nondevel-make", "make -j"),
+ ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
+ ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
+ ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
+ ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
+ ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
+ ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
+ ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
+ ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
+ ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
+ ("nondevel-no-public-nss_winbind",
+ check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
+ ("nondevel-no-public-nss_wins",
+ check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
+ ("nondevel-no-public-libwbclient",
+ check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
+ ("nondevel-no-public-pam_winbind",
+ check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
+ ("nondevel-no-public-winbind_krb5_locator",
+ check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
+ ("nondevel-no-public-async_dns_krb5_locator",
+ check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
+ ("nondevel-install", "make -j install"),
+ ("nondevel-dist", "make dist"),
+
+ ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
+ ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
+ ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
+ ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
+ ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
+ ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
+ ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
+ ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
+ ("prefix-no-public-nss_winbind",
+ check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
+ ("prefix-no-public-nss_wins",
+ check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
+ ("prefix-no-public-libwbclient",
+ check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
+ ("prefix-no-public-pam_winbind",
+ check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
+ ("prefix-no-public-winbind_krb5_locator",
+ check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
+ "service_locator")),
+ ("prefix-no-public-async_dns_krb5_locator",
+ check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
+ "service_locator")),
+
+ # retry with all modules shared
+ ("allshared-distclean", "make distclean"),
+ ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
+ ("allshared-make", "make -j"),
+ ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
+ ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
+ ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
+ ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
+ ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
+ ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
+ ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
+ ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
+ ("allshared-no-public-nss_winbind",
+ check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
+ ("allshared-no-public-nss_wins",
+ check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
+ ("allshared-no-public-libwbclient",
+ check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
+ ("allshared-no-public-pam_winbind",
+ check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
+ ("allshared-no-public-winbind_krb5_locator",
+ check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
+ ("allshared-no-public-async_dns_krb5_locator",
+ check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
+ ],
+ },
+
+ "samba-fuzz": {
+ "sequence": [
+ # build the fuzzers (static) via the oss-fuzz script
+ ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
+ ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
+ ],
+ },
+
+ # * Test smbd and smbtorture can build semi-static
+ #
+ # * Test Samba without python still builds.
+ #
+ # When this test fails due to more use of Python, the expectations
+ # is that the newly failing part of the code should be disabled
+ # when --disable-python is set (rather than major work being done
+ # to support this environment).
+ #
+ # The target here is for vendors shipping a minimal smbd.
+ "samba-minimal-smbd": {
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+
+ # build with all modules static
+ ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
+ ("allstatic-make", "make -j"),
+ ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
+ ("allstatic-lcov", LCOV_CMD),
+ ("allstatic-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("allstatic-def-clean", "make clean"),
+
+ # force all libraries as private
+ ("allprivate-def-distclean", "make distclean"),
+ ("allprivate-def-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL"),
+ ("allprivate-def-make", "make -j"),
+ # note wrapper libraries need to be public
+ ("allprivate-def-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so' | wc -l | grep -q '^0'"),
+ ("allprivate-def-only-private-ext", "ls ./bin/shared/private | egrep 'private-samba' | wc -l | grep -q '^0' && exit 1; exit 0"),
+ ("allprivate-def-no-non-private-ext", "ls ./bin/shared/private | egrep -v 'private-samba|^libpypamtest.so$' | wc -l | grep -q '^0'"),
+ ("allprivate-def-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
+ ("allprivate-def-lcov", LCOV_CMD),
+ ("allprivate-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("allprivate-def-clean", "make clean"),
+
+ # force all libraries as private with a non default
+ # extension and 2 exceptions
+ ("allprivate-ext-distclean", "make distclean"),
+ ("allprivate-ext-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL --private-library-extension=private-library --private-extension-exception=pac,ndr"),
+ ("allprivate-ext-make", "make -j"),
+ # note wrapper libraries need to be public
+ ("allprivate-ext-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so' | wc -l | grep -q '^0'"),
+ ("allprivate-ext-no-private-default-ext", "ls ./bin/shared/private | grep 'private-samba' | wc -l | grep -q '^0'"),
+ ("allprivate-ext-has-private-ext", "ls ./bin/shared/private | grep 'private-library' | wc -l | grep -q '^0' && exit 1; exit 0"),
+ ("allprivate-ext-libndr-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libndr' | wc -l | grep -q '^1'"),
+ ("allprivate-ext-libpac-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libpac' | wc -l | grep -q '^1'"),
+ ("allprivate-ext-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
+ ("allprivate-ext-lcov", LCOV_CMD),
+ ("allprivate-ext-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("allprivate-ext-clean", "make clean"),
+
+ # retry with nonshared smbd and smbtorture
+ ("nonshared-distclean", "make distclean"),
+ ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
+ ("nonshared-make", "make -j"),
+ ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
+ ("nonshared-lcov", LCOV_CMD),
+ ("nonshared-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("nonshared-clean", "make clean"),
+ ],
+ },
+
+ "samba-nopython": {
+ "sequence": [
+ ("random-sleep", random_sleep(300, 900)),
+
+ ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
+ ("make", "make -j"),
+ ("find-python", "script/find_python.sh ${PREFIX}"),
+ ("test", "make test-nopython"),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("clean", "make clean"),
+
+ ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
+ ("talloc-make", "cd lib/talloc && make"),
+ ("talloc-install", "cd lib/talloc && make install"),
+
+ ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
+ ("tdb-make", "cd lib/tdb && make"),
+ ("tdb-install", "cd lib/tdb && make install"),
+
+ ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
+ ("tevent-make", "cd lib/tevent && make"),
+ ("tevent-install", "cd lib/tevent && make install"),
+
+ ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
+ ("ldb-make", "cd lib/ldb && make"),
+ ("ldb-install", "cd lib/ldb && make install"),
+
+ # retry against installed library packages, but no required modules
+ ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
+ ("libs-make", "make -j"),
+ ("libs-install", "make install"),
+ ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("libs-clean", "make clean"),
+
+ ],
+ },
+
+ "samba-codecheck": {
+ "sequence": [
+ ("run", "script/check-shell-scripts.sh ."),
+ ("run", "script/codespell.sh ."),
+ ],
+ },
+
+ "ldb": {
+ "sequence": [
+ ("random-sleep", random_sleep(60, 600)),
+ ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+ ("make", "make"),
+ ("install", "make install"),
+ ("test", "make test"),
+ ("lcov", LCOV_CMD),
+ ("clean", "make clean"),
+ ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
+ ("make-no-lmdb", "make"),
+ ("test-no-lmdb", "make test"),
+ ("lcov-no-lmdb", LCOV_CMD),
+ ("install-no-lmdb", "make install"),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("distcheck", "make distcheck"),
+ ("clean", "make clean"),
+ ],
+ },
+
+ "tdb": {
+ "sequence": [
+ ("random-sleep", random_sleep(60, 600)),
+ ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+ ("make", "make"),
+ ("install", "make install"),
+ ("test", "make test"),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("distcheck", "make distcheck"),
+ ("clean", "make clean"),
+ ],
+ },
+
+ "talloc": {
+ "sequence": [
+ ("random-sleep", random_sleep(60, 600)),
+ ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+ ("make", "make"),
+ ("install", "make install"),
+ ("test", "make test"),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("distcheck", "make distcheck"),
+ ("clean", "make clean"),
+ ],
+ },
+
+ "replace": {
+ "sequence": [
+ ("random-sleep", random_sleep(60, 600)),
+ ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+ ("make", "make"),
+ ("install", "make install"),
+ ("test", "make test"),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("distcheck", "make distcheck"),
+ ("clean", "make clean"),
+ ],
+ },
+
+ "tevent": {
+ "sequence": [
+ ("random-sleep", random_sleep(60, 600)),
+ ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+ ("make", "make"),
+ ("install", "make install"),
+ ("test", "make test"),
+ ("lcov", LCOV_CMD),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("distcheck", "make distcheck"),
+ ("clean", "make clean"),
+ ],
+ },
+
+ "pidl": {
+ "git-clone-required": True,
+ "sequence": [
+ ("random-sleep", random_sleep(60, 600)),
+ ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
+ ("touch", "touch *.yp"),
+ ("make", "make"),
+ ("test", "make test"),
+ ("install", "make install"),
+ ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
+ ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+ ("clean", "make clean"),
+ ],
+ },
+
+ # these are useful for debugging autobuild
+ "pass": {
+ "sequence": [
+ ("pass", 'echo passing && /bin/true'),
+ ],
+ },
+ "fail": {
+ "sequence": [
+ ("fail", 'echo failing && /bin/false'),
+ ],
+ },
+}
+
+defaulttasks = list(tasks.keys())
+
+defaulttasks.remove("pass")
+defaulttasks.remove("fail")
+
+# The build tasks will be brought in by the test tasks as needed
+defaulttasks.remove("samba-def-build")
+defaulttasks.remove("samba-nt4-build")
+defaulttasks.remove("samba-mit-build")
+defaulttasks.remove("samba-h5l-build")
+defaulttasks.remove("samba-no-opath-build")
+
+# This is not a normal test, but a task to support manually running
+# one test under autobuild
+defaulttasks.remove("samba-test-only")
+
+# Only built on GitLab CI and not in the default autobuild because it
+# uses too much space (4GB of semi-static binaries)
+defaulttasks.remove("samba-fuzz")
+
+# The FIPS build runs only in GitLab CI on a current Fedora Docker
+# container where a simulated FIPS mode is possible.
+defaulttasks.remove("samba-fips")
+
+# The MIT build runs on a current Fedora where an up to date MIT KDC
+# is already packaged. This avoids needing to backport a current MIT
+# to the default Ubuntu 18.04, particularly during development, and
+# the need to install on the shared sn-devel-184.
+
+defaulttasks.remove("samba-mitkrb5")
+defaulttasks.remove("samba-admem-mit")
+defaulttasks.remove("samba-addc-mit-1")
+defaulttasks.remove("samba-addc-mit-4a")
+defaulttasks.remove("samba-addc-mit-4b")
+
+defaulttasks.remove("samba-32bit")
+
+if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
+ defaulttasks.remove("samba-o3")
+
+
+def do_print(msg):
+ logger.info(msg)
+ sys.stdout.flush()
+ sys.stderr.flush()
+
+def do_debug(msg):
+ logger.debug(msg)
+ sys.stdout.flush()
+ sys.stderr.flush()
+
+
+def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
+ if show is None:
+ do_debug("Running: '%s' in '%s'" % (cmd, dir))
+ elif show:
+ do_print("Running: '%s' in '%s'" % (cmd, dir))
+
+ if output:
+ out = check_output([cmd], shell=True, cwd=dir)
+ return out.decode(encoding='utf-8', errors='backslashreplace')
+ elif checkfail:
+ return check_call(cmd, shell=True, cwd=dir)
+ else:
+ return call(cmd, shell=True, cwd=dir)
+
+def rmdir_force(dirname, re_raise=True):
+ try:
+ run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
+ dirname, dirname, dirname), output=True, show=True)
+ except CalledProcessError as e:
+ do_print("Failed: '%s'" % (str(e)))
+ run_cmd("tree %s" % dirname, output=True, show=True)
+ if re_raise:
+ raise
+ return False
+ return True
+
+class builder(object):
+ '''handle build of one directory'''
+
+ def __init__(self, name, definition):
+ self.name = name
+ self.dir = builddirs.get(name, '.')
+ self.tag = self.name.replace('/', '_')
+ self.definition = definition
+ self.sequence = definition["sequence"]
+ self.git_clone_required = False
+ if "git-clone-required" in definition:
+ self.git_clone_required = bool(definition["git-clone-required"])
+ self.proc = None
+ self.done = False
+ self.next = 0
+ self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
+ self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
+ do_debug("stdout for %s in %s" % (self.name, self.stdout_path))
+ do_debug("stderr for %s in %s" % (self.name, self.stderr_path))
+ run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
+ self.stdout = open(self.stdout_path, 'w')
+ self.stderr = open(self.stderr_path, 'w')
+ self.stdin = open("/dev/null", 'r')
+ self.builder_dir = "%s/%s" % (testbase, self.tag)
+ self.test_source_dir = self.builder_dir
+ self.cwd = "%s/%s" % (self.builder_dir, self.dir)
+ self.selftest_prefix = "%s/bin/ab" % (self.cwd)
+ self.prefix = "%s/%s" % (test_prefix, self.tag)
+ self.consumers = []
+ self.producer = None
+
+ if self.git_clone_required:
+ assert "dependency" not in definition
+
+ def mark_existing(self):
+ do_debug('%s: Mark as existing dependency' % self.name)
+ self.next = len(self.sequence)
+ self.done = True
+
+ def add_consumer(self, consumer):
+ do_debug("%s: add consumer: %s" % (self.name, consumer.name))
+ consumer.producer = self
+ consumer.test_source_dir = self.test_source_dir
+ self.consumers.append(consumer)
+
+ def start_next(self):
+ if self.producer is not None:
+ if not self.producer.done:
+ do_debug("%s: Waiting for producer: %s" % (self.name, self.producer.name))
+ return
+
+ if self.next == 0:
+ rmdir_force(self.builder_dir)
+ rmdir_force(self.prefix)
+ if self.producer is not None:
+ run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
+ elif not self.git_clone_required:
+ run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
+ else:
+ run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
+
+ if self.next == len(self.sequence):
+ if not self.done:
+ do_print('%s: Completed OK' % self.name)
+ self.done = True
+ if not options.nocleanup and len(self.consumers) == 0:
+ do_print('%s: Cleaning up' % self.name)
+ rmdir_force(self.builder_dir)
+ rmdir_force(self.prefix)
+ for consumer in self.consumers:
+ if consumer.next != 0:
+ continue
+ do_print('%s: Starting consumer %s' % (self.name, consumer.name))
+ consumer.start_next()
+ if self.producer is not None:
+ self.producer.consumers.remove(self)
+ assert self.producer.done
+ self.producer.start_next()
+ do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
+ return
+ (self.stage, self.cmd) = self.sequence[self.next]
+ self.cmd = self.cmd.replace("${PYTHON_PREFIX}",
+ get_path(name='platlib',
+ scheme="posix_prefix",
+ vars={"base": self.prefix,
+ "platbase": self.prefix}))
+ self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
+ self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
+ self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
+ self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
+ self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
+ self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
+ self.cmd = self.cmd.replace("${NAME}", self.name)
+ self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
+ do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
+ self.proc = Popen(self.cmd, shell=True,
+ close_fds=True, cwd=self.cwd,
+ stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
+ self.next += 1
+
+def expand_dependencies(n):
+ deps = list()
+ if "dependency" in tasks[n]:
+ depname = tasks[n]["dependency"]
+ assert depname in tasks
+ sdeps = expand_dependencies(depname)
+ assert n not in sdeps
+ for sdep in sdeps:
+ deps.append(sdep)
+ deps.append(depname)
+ return deps
+
+
+class buildlist(object):
+ '''handle build of multiple directories'''
+
+ def __init__(self, tasknames, rebase_url, rebase_branch="master"):
+ self.tail_proc = None
+ self.retry = None
+ if not tasknames:
+ if options.restrict_tests:
+ tasknames = ["samba-test-only"]
+ else:
+ tasknames = defaulttasks
+
+ given_tasknames = tasknames.copy()
+ implicit_tasknames = []
+ for n in given_tasknames:
+ deps = expand_dependencies(n)
+ for dep in deps:
+ if dep in given_tasknames:
+ continue
+ if dep in implicit_tasknames:
+ continue
+ implicit_tasknames.append(dep)
+
+ tasknames = implicit_tasknames.copy()
+ tasknames.extend(given_tasknames)
+ do_debug("given_tasknames: %s" % given_tasknames)
+ do_debug("implicit_tasknames: %s" % implicit_tasknames)
+ do_debug("tasknames: %s" % tasknames)
+ self.tlist = [builder(n, tasks[n]) for n in tasknames]
+
+ if options.retry:
+ rebase_remote = "rebaseon"
+ retry_task = {
+ "git-clone-required": True,
+ "sequence": [
+ ("retry",
+ '''set -e
+ git remote add -t %s %s %s
+ git fetch %s
+ while :; do
+ sleep 60
+ git describe %s/%s > old_remote_branch.desc
+ git fetch %s
+ git describe %s/%s > remote_branch.desc
+ diff old_remote_branch.desc remote_branch.desc
+ done
+ ''' % (
+ rebase_branch, rebase_remote, rebase_url,
+ rebase_remote,
+ rebase_remote, rebase_branch,
+ rebase_remote,
+ rebase_remote, rebase_branch
+ ))]}
+
+ self.retry = builder('retry', retry_task)
+ self.need_retry = False
+
+ if options.skip_dependencies:
+ for b in self.tlist:
+ if b.name in implicit_tasknames:
+ b.mark_existing()
+
+ for b in self.tlist:
+ do_debug("b.name=%s" % b.name)
+ if "dependency" not in b.definition:
+ continue
+ depname = b.definition["dependency"]
+ do_debug("b.name=%s: dependency:%s" % (b.name, depname))
+ for p in self.tlist:
+ if p.name == depname:
+ p.add_consumer(b)
+
+ def kill_kids(self):
+ if self.tail_proc is not None:
+ self.tail_proc.terminate()
+ self.tail_proc.wait()
+ self.tail_proc = None
+ if self.retry is not None:
+ self.retry.proc.terminate()
+ self.retry.proc.wait()
+ self.retry = None
+ for b in self.tlist:
+ if b.proc is not None:
+ run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
+ b.proc.terminate()
+ b.proc.wait()
+ b.proc = None
+
+ def wait_one(self):
+ while True:
+ none_running = True
+ for b in self.tlist:
+ if b.proc is None:
+ continue
+ none_running = False
+ b.status = b.proc.poll()
+ if b.status is None:
+ continue
+ b.proc = None
+ return b
+ if options.retry:
+ ret = self.retry.proc.poll()
+ if ret is not None:
+ self.need_retry = True
+ self.retry = None
+ return None
+ if none_running:
+ return None
+ time.sleep(0.1)
+
+ def run(self):
+ for b in self.tlist:
+ b.start_next()
+ if options.retry:
+ self.retry.start_next()
+ while True:
+ b = self.wait_one()
+ if options.retry and self.need_retry:
+ self.kill_kids()
+ do_print("retry needed")
+ return (0, None, None, None, "retry")
+ if b is None:
+ break
+ if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
+ self.kill_kids()
+ return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
+ b.start_next()
+ self.kill_kids()
+ return (0, None, None, None, "All OK")
+
+ def write_system_info(self, filename):
+ with open(filename, 'w') as f:
+ for cmd in ['uname -a',
+ 'lsb_release -a',
+ 'free',
+ 'mount',
+ 'cat /proc/cpuinfo',
+ 'cc --version',
+ 'df -m .',
+ 'df -m %s' % testbase]:
+ try:
+ out = run_cmd(cmd, output=True, checkfail=False)
+ except CalledProcessError as e:
+ out = "<failed: %s>" % str(e)
+ print('### %s' % cmd, file=f)
+ print(out, file=f)
+ print(file=f)
+
+ def tarlogs(self, fname):
+ with tarfile.open(fname, "w:gz") as tar:
+ for b in self.tlist:
+ tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
+ tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
+ if os.path.exists("autobuild.log"):
+ tar.add("autobuild.log")
+ filename = 'system-info.txt'
+ self.write_system_info(filename)
+ tar.add(filename)
+
+ def remove_logs(self):
+ for b in self.tlist:
+ os.unlink(b.stdout_path)
+ os.unlink(b.stderr_path)
+
+ def start_tail(self):
+ cmd = ["tail", "-f"]
+ for b in self.tlist:
+ cmd.append(b.stdout_path)
+ cmd.append(b.stderr_path)
+ self.tail_proc = Popen(cmd, close_fds=True)
+
+
+def cleanup(do_raise=False):
+ if options.nocleanup:
+ return
+ run_cmd("stat %s || true" % test_tmpdir, show=True)
+ run_cmd("stat %s" % testbase, show=True)
+ do_print("Cleaning up %r" % cleanup_list)
+ for d in cleanup_list:
+ ok = rmdir_force(d, re_raise=False)
+ if ok:
+ continue
+ if os.path.isdir(d):
+ do_print("Killing, waiting and retry")
+ run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
+ else:
+ do_print("Waiting and retry")
+ time.sleep(1)
+ rmdir_force(d, re_raise=do_raise)
+
+
+def daemonize(logfile):
+ pid = os.fork()
+ if pid == 0: # Parent
+ os.setsid()
+ pid = os.fork()
+ if pid != 0: # Actual daemon
+ os._exit(0)
+ else: # Grandparent
+ os._exit(0)
+
+ import resource # Resource usage information.
+ maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+ if maxfd == resource.RLIM_INFINITY:
+ maxfd = 1024 # Rough guess at maximum number of open file descriptors.
+ for fd in range(0, maxfd):
+ try:
+ os.close(fd)
+ except OSError:
+ pass
+ os.open(logfile, os.O_RDWR | os.O_CREAT)
+ os.dup2(0, 1)
+ os.dup2(0, 2)
+
+
+def write_pidfile(fname):
+ '''write a pid file, cleanup on exit'''
+ with open(fname, mode='w') as f:
+ f.write("%u\n" % os.getpid())
+
+
+def rebase_tree(rebase_url, rebase_branch="master"):
+ rebase_remote = "rebaseon"
+ do_print("Rebasing on %s" % rebase_url)
+ run_cmd("git describe HEAD", show=True, dir=test_master)
+ run_cmd("git remote add -t %s %s %s" %
+ (rebase_branch, rebase_remote, rebase_url),
+ show=True, dir=test_master)
+ run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
+ if options.fix_whitespace:
+ run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
+ (rebase_remote, rebase_branch),
+ show=True, dir=test_master)
+ else:
+ run_cmd("git rebase --force-rebase %s/%s" %
+ (rebase_remote, rebase_branch),
+ show=True, dir=test_master)
+ diff = run_cmd("git --no-pager diff HEAD %s/%s" %
+ (rebase_remote, rebase_branch),
+ dir=test_master, output=True)
+ if diff == '':
+ do_print("No differences between HEAD and %s/%s - exiting" %
+ (rebase_remote, rebase_branch))
+ sys.exit(0)
+ run_cmd("git describe %s/%s" %
+ (rebase_remote, rebase_branch),
+ show=True, dir=test_master)
+ run_cmd("git describe HEAD", show=True, dir=test_master)
+ run_cmd("git --no-pager diff --stat HEAD %s/%s" %
+ (rebase_remote, rebase_branch),
+ show=True, dir=test_master)
+
+
+def push_to(push_url, push_branch="master"):
+ push_remote = "pushto"
+ do_print("Pushing to %s" % push_url)
+ if options.mark:
+ run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
+ run_cmd("git commit --amend -c HEAD", dir=test_master)
+ # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
+ # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
+ run_cmd("git remote add -t %s %s %s" %
+ (push_branch, push_remote, push_url),
+ show=True, dir=test_master)
+ run_cmd("git push %s +HEAD:%s" %
+ (push_remote, push_branch),
+ show=True, dir=test_master)
+
+
+def send_email(subject, text, log_tar):
+ if options.email is None:
+ do_print("not sending email because the recipient is not set")
+ do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
+ (subject, text))
+ return
+ outer = MIMEMultipart()
+ outer['Subject'] = subject
+ outer['To'] = options.email
+ outer['From'] = options.email_from
+ outer['Date'] = email.utils.formatdate(localtime=True)
+ outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
+ outer.attach(MIMEText(text, 'plain', 'utf-8'))
+ if options.attach_logs:
+ with open(log_tar, 'rb') as fp:
+ msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
+ # Set the filename parameter
+ msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
+ outer.attach(msg)
+ content = outer.as_string()
+ s = smtplib.SMTP(options.email_server)
+ email_user = os.getenv('SMTP_USERNAME')
+ email_password = os.getenv('SMTP_PASSWORD')
+ if email_user is not None:
+ s.starttls()
+ s.login(email_user, email_password)
+
+ s.sendmail(options.email_from, [options.email], content)
+ s.set_debuglevel(1)
+ s.quit()
+
+
+def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
+ elapsed_time, log_base=None, add_log_tail=True):
+ '''send an email to options.email about the failure'''
+ elapsed_minutes = elapsed_time / 60.0
+ if log_base is None:
+ log_base = gitroot
+ text = '''
+Dear Developer,
+
+Your autobuild on %s failed after %.1f minutes
+when trying to test %s with the following error:
+
+ %s
+
+the autobuild has been abandoned. Please fix the error and resubmit.
+
+A summary of the autobuild process is here:
+
+ %s/autobuild.log
+''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
+
+ if options.restrict_tests:
+ text += """
+The build was restricted to tests matching %s\n""" % options.restrict_tests
+
+ if failed_task != 'rebase':
+ text += '''
+You can see logs of the failed task here:
+
+ %s/%s.stdout
+ %s/%s.stderr
+
+or you can get full logs of all tasks in this job here:
+
+ %s/logs.tar.gz
+
+The top commit for the tree that was built was:
+
+%s
+
+''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
+
+ log_stdout = "%s/%s.stdout" % (gitroot, failed_tag)
+ if add_log_tail and os.access(log_stdout, os.R_OK):
+ f = open(log_stdout, 'r')
+ lines = f.readlines()
+ log_tail = "".join(lines[-50:])
+ num_lines = len(lines)
+ log_stderr = "%s/%s.stderr" % (gitroot, failed_tag)
+ if num_lines < 50 and os.access(log_stderr, os.R_OK):
+ # Also include stderr (compile failures) if < 50 lines of stdout
+ f = open(log_stderr, 'r')
+ log_tail += "".join(f.readlines()[-(50 - num_lines):])
+
+ text += '''
+The last 50 lines of log messages:
+
+%s
+ ''' % log_tail
+ f.close()
+
+ logs = os.path.join(gitroot, 'logs.tar.gz')
+ send_email('autobuild[%s] failure on %s for task %s during %s'
+ % (options.branch, platform.node(), failed_task, failed_stage),
+ text, logs)
+
+
+def email_success(elapsed_time, log_base=None):
+ '''send an email to options.email about a successful build'''
+ if log_base is None:
+ log_base = gitroot
+ text = '''
+Dear Developer,
+
+Your autobuild on %s has succeeded after %.1f minutes.
+
+''' % (platform.node(), elapsed_time / 60.)
+
+ if options.restrict_tests:
+ text += """
+The build was restricted to tests matching %s\n""" % options.restrict_tests
+
+ if options.keeplogs:
+ text += '''
+
+you can get full logs of all tasks in this job here:
+
+ %s/logs.tar.gz
+
+''' % log_base
+
+ text += '''
+The top commit for the tree that was built was:
+
+%s
+''' % top_commit_msg
+
+ logs = os.path.join(gitroot, 'logs.tar.gz')
+ send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
+ text, logs)
+
+
+# get the top commit message, for emails
+top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
+
+try:
+ if options.skip_dependencies:
+ run_cmd("stat %s" % testbase, dir=testbase, output=True)
+ else:
+ os.makedirs(testbase)
+except Exception as reason:
+ raise Exception("Unable to create %s : %s" % (testbase, reason))
+cleanup_list.append(testbase)
+
+if options.daemon:
+ logfile = os.path.join(testbase, "log")
+ do_print("Forking into the background, writing progress to %s" % logfile)
+ daemonize(logfile)
+
+write_pidfile(gitroot + "/autobuild.pid")
+
+start_time = time.time()
+
+while True:
+ try:
+ run_cmd("rm -rf %s" % test_tmpdir, show=True)
+ os.makedirs(test_tmpdir)
+ # The waf uninstall code removes empty directories all the way
+ # up the tree. Creating a file in test_tmpdir stops it from
+ # being removed.
+ run_cmd("touch %s" % os.path.join(test_tmpdir,
+ ".directory-is-not-empty"), show=True)
+ run_cmd("stat %s" % test_tmpdir, show=True)
+ run_cmd("stat %s" % testbase, show=True)
+ if options.skip_dependencies:
+ run_cmd("stat %s" % test_master, dir=testbase, output=True)
+ else:
+ run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
+ except Exception:
+ cleanup()
+ raise
+
+ try:
+ if options.rebase is not None:
+ rebase_tree(options.rebase, rebase_branch=options.branch)
+ except Exception:
+ cleanup_list.append(gitroot + "/autobuild.pid")
+ cleanup()
+ elapsed_time = time.time() - start_time
+ email_failure(-1, 'rebase', 'rebase', 'rebase',
+ 'rebase on %s failed' % options.branch,
+ elapsed_time, log_base=options.log_base)
+ sys.exit(1)
+
+ try:
+ blist = buildlist(args, options.rebase, rebase_branch=options.branch)
+ if options.tail:
+ blist.start_tail()
+ (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
+ if status != 0 or errstr != "retry":
+ break
+ cleanup(do_raise=True)
+ except Exception:
+ cleanup()
+ raise
+
+cleanup_list.append(gitroot + "/autobuild.pid")
+
+do_print(errstr)
+
+blist.kill_kids()
+if options.tail:
+ do_print("waiting for tail to flush")
+ time.sleep(1)
+
+elapsed_time = time.time() - start_time
+if status == 0:
+ if options.passcmd is not None:
+ do_print("Running passcmd: %s" % options.passcmd)
+ run_cmd(options.passcmd, dir=test_master)
+ if options.pushto is not None:
+ push_to(options.pushto, push_branch=options.branch)
+ if options.keeplogs or options.attach_logs:
+ blist.tarlogs("logs.tar.gz")
+ do_print("Logs in logs.tar.gz")
+ if options.always_email:
+ email_success(elapsed_time, log_base=options.log_base)
+ blist.remove_logs()
+ cleanup()
+ do_print(errstr)
+ sys.exit(0)
+
+# something failed, gather a tar of the logs
+blist.tarlogs("logs.tar.gz")
+
+if options.email is not None:
+ email_failure(status, failed_task, failed_stage, failed_tag, errstr,
+ elapsed_time, log_base=options.log_base)
+else:
+ elapsed_minutes = elapsed_time / 60.0
+ print('''
+
+####################################################################
+
+AUTOBUILD FAILURE
+
+Your autobuild[%s] on %s failed after %.1f minutes
+when trying to test %s with the following error:
+
+ %s
+
+the autobuild has been abandoned. Please fix the error and resubmit.
+
+####################################################################
+
+''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
+
+cleanup()
+do_print(errstr)
+do_print("Logs in logs.tar.gz")
+sys.exit(status)