Adding upstream version 2:4.20.0+dfsg.upstream/2%4.20.0+dfsgupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

25 files changed, 5671 insertions, 0 deletions

diff --git a/script/attr_count_read b/script/attr_count_read
new file mode 100755
index 0000000..2f6a4d0
--- /dev/null
+++ b/script/attr_count_read
@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) Catalyst IT Ltd. 2019
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys
+import argparse
+import struct
+import os
+from collections import OrderedDict, Counter
+from pprint import pprint
+
+sys.path.insert(0, "bin/python")
+import tdb
+
+
+def unpack_uint(filename, casefold=True):
+    db = tdb.Tdb(filename)
+    d = {}
+    for k in db:
+        v = struct.unpack("I", db[k])[0]
+        k2 = k.decode('utf-8')
+        if casefold:
+            k2 = k2.lower()
+        if k2 in d: # because casefold
+            d[k2] += v
+        else:
+            d[k2] = v
+    return d
+
+
+def unpack_ssize_t_pair(filename, casefold):
+    db = tdb.Tdb(filename)
+    pairs = []
+    for k in db:
+        key = struct.unpack("nn", k)
+        v = struct.unpack("I", db[k])[0]
+        pairs.append((v, key))
+
+    pairs.sort(reverse=True)
+    #print(pairs)
+    return [(k, v) for (v, k) in pairs]
+
+
+DATABASES = [
+    ('requested', "debug/attr_counts_requested.tdb", unpack_uint,
+     "The attribute was specifically requested."),
+    ('duplicates', "debug/attr_counts_duplicates.tdb", unpack_uint,
+     "Requested more than once in the same request."),
+    ('empty request', "debug/attr_counts_empty_req.tdb", unpack_uint,
+     "No attributes were requested, but these were returned"),
+    ('null request', "debug/attr_counts_null_req.tdb", unpack_uint,
+     "The attribute list was NULL and these were returned."),
+    ('found', "debug/attr_counts_found.tdb", unpack_uint,
+     "The attribute was specifically requested and it was found."),
+    ('not found', "debug/attr_counts_not_found.tdb", unpack_uint,
+     "The attribute was specifically requested but was not found."),
+    ('unwanted', "debug/attr_counts_unwanted.tdb", unpack_uint,
+     "The attribute was not requested and it was found."),
+    ('star match', "debug/attr_counts_star_match.tdb", unpack_uint,
+     'The attribute was not specifically requested but "*" was.'),
+    ('req vs found', "debug/attr_counts_req_vs_found.tdb", unpack_ssize_t_pair,
+     "How many attributes were requested versus how many were returned."),
+]
+
+
+def plot_pair_data(name, data, doc, lim=90):
+    # Note we keep the matplotlib import internal to this function for
+    # two reasons:
+    # 1. Some people won't have matplotlib, but might want to run the
+    #    script.
+    # 2. The import takes hundreds of milliseconds, which is a
+    #    nuisance if you don't want graphs.
+    #
+    # This plot could be improved!
+    import matplotlib.pylab as plt
+    fig, ax = plt.subplots()
+    if lim:
+        data2 = []
+        for p, c in data:
+            if p[0] > lim or p[1] > lim:
+                print("not plotting %s: %s" % (p, c))
+                continue
+            data2.append((p, c))
+        skipped = len(data) - len(data2)
+        if skipped:
+            name += " (excluding %d out of range values)" % skipped
+            data = data2
+    xy, counts = zip(*data)
+    x, y = zip(*xy)
+    bins_x = max(x) + 4
+    bins_y = max(y)
+    ax.set_title(name)
+    ax.scatter(x, y, c=counts)
+    plt.show()
+
+
+def print_pair_data(name, data, doc):
+    print(name)
+    print(doc)
+    t = "%14s | %14s | %14s"
+    print(t % ("requested", "returned", "count"))
+    print(t % (('-' * 14,) * 3))
+
+    for xy, count in data:
+        x, y = xy
+        if x == -2:
+            x = 'NULL'
+        elif x == -4:
+            x = '*'
+        print(t % (x, y, count))
+
+
+def print_counts(count_data):
+    all_attrs = Counter()
+    for c in count_data:
+        all_attrs.update(c[1])
+
+    print("found %d attrs" % len(all_attrs))
+    longest = max(len(x) for x in all_attrs)
+
+    #pprint(all_attrs)
+    rows = OrderedDict()
+    for a, _ in all_attrs.most_common():
+        rows[a] = [a]
+
+    for col_name, counts, doc in count_data:
+        for attr, row in rows.items():
+            d = counts.get(attr, '')
+            row.append(d)
+
+        print("%15s: %s" % (col_name, doc))
+    print()
+
+    t = "%{}s".format(longest)
+    for c in count_data:
+        t += " | %{}s".format(max(len(c[0]), 7))
+
+    h = t % (("attribute",) + tuple(c[0] for c in count_data))
+    print(h)
+    print("-" * len(h))
+
+    for attr, row in rows.items():
+        print(t % tuple(row))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('LDB_PRIVATE_DIR',
+                        help="read attr counts in this directory")
+    parser.add_argument('--plot', action="store_true",
+                        help='attempt to draw graphs')
+    parser.add_argument('--no-casefold', action="store_false",
+                        default=True, dest="casefold",
+                        help='See all the encountered case variants')
+    args = parser.parse_args()
+
+    if not os.path.isdir(args.LDB_PRIVATE_DIR):
+        parser.print_usage()
+        sys.exit(1)
+
+    count_data = []
+    pair_data = []
+    for k, filename, unpacker, doc in DATABASES:
+        filename = os.path.join(args.LDB_PRIVATE_DIR, filename)
+        try:
+            d = unpacker(filename, casefold=args.casefold)
+        except (RuntimeError, IOError) as e:
+            print("could not parse %s: %s" % (filename, e))
+            continue
+        if unpacker is unpack_ssize_t_pair:
+            pair_data.append((k, d, doc))
+        else:
+            count_data.append((k, d, doc))
+
+    for k, v, doc in pair_data:
+        if args.plot:
+            plot_pair_data(k, v, doc)
+        print_pair_data(k, v, doc)
+
+    print()
+    print_counts(count_data)
+
+main()
diff --git a/script/autobuild.py b/script/autobuild.py
new file mode 100755
index 0000000..ecec352
--- /dev/null
+++ b/script/autobuild.py
@@ -0,0 +1,1902 @@
+#!/usr/bin/env python3
+# run tests on all Samba subprojects and push to a git tree on success
+# Copyright Andrew Tridgell 2010
+# released under GNU GPL v3 or later
+
+from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
+import os
+import tarfile
+import sys
+import time
+import random
+from optparse import OptionParser
+import smtplib
+import email
+from email.mime.text import MIMEText
+from email.mime.base import MIMEBase
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from sysconfig import get_path
+import platform
+
+import logging
+
+try:
+    from waflib.Build import CACHE_SUFFIX
+except ImportError:
+    sys.path.insert(0, "./third_party/waf")
+    from waflib.Build import CACHE_SUFFIX
+
+logging.basicConfig(format='%(asctime)s %(message)s')
+logger = logging.getLogger('autobuild')
+logger.setLevel(logging.INFO)
+
+os.environ["PYTHONUNBUFFERED"] = "1"
+
+# This speeds up testing remarkably.
+os.environ['TDB_NO_FSYNC'] = '1'
+
+# allow autobuild to run within git rebase -i
+if "GIT_DIR" in os.environ:
+    del os.environ["GIT_DIR"]
+if "GIT_WORK_TREE" in os.environ:
+    del os.environ["GIT_WORK_TREE"]
+
+def find_git_root():
+    '''get to the top of the git repo'''
+    p = os.getcwd()
+    while p != '/':
+        if os.path.exists(os.path.join(p, ".git")):
+            return p
+        p = os.path.abspath(os.path.join(p, '..'))
+    return None
+
+
+gitroot = find_git_root()
+if gitroot is None:
+    raise Exception("Failed to find git root")
+
+
+def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
+
+parser = OptionParser()
+parser.add_option("--tail", help="show output while running", default=False, action="store_true")
+parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
+parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
+parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
+parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
+                  default=def_testbase)
+parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
+                  default=None)
+parser.add_option("--passcmd", help="command to run on success", default=None)
+parser.add_option("--verbose", help="show all commands as they are run",
+                  default=False, action="store_true")
+parser.add_option("--rebase", help="rebase on the given tree before testing",
+                  default=None, type='str')
+parser.add_option("--pushto", help="push to a git url on success",
+                  default=None, type='str')
+parser.add_option("--mark", help="add a Tested-By signoff before pushing",
+                  default=False, action="store_true")
+parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
+                  default=False, action="store_true")
+parser.add_option("--retry", help="automatically retry if master changes",
+                  default=False, action="store_true")
+parser.add_option("--email", help="send email to the given address on failure",
+                  type='str', default=None)
+parser.add_option("--email-from", help="send email from the given address",
+                  type='str', default="autobuild@samba.org")
+parser.add_option("--email-server", help="send email via the given server",
+                  type='str', default='localhost')
+parser.add_option("--always-email", help="always send email, even on success",
+                  action="store_true")
+parser.add_option("--daemon", help="daemonize after initial setup",
+                  action="store_true")
+parser.add_option("--branch", help="the branch to work on (default=master)",
+                  default="master", type='str')
+parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
+                  default=gitroot, type='str')
+parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
+                  default=False, action="store_true")
+parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
+                  default='')
+parser.add_option("--enable-coverage", dest='enable_coverage',
+                  action="store_const", const='--enable-coverage', default='',
+                  help="Add --enable-coverage option while configure")
+
+(options, args) = parser.parse_args()
+
+if options.retry:
+    if options.rebase is None:
+        raise Exception('You can only use --retry if you also rebase')
+
+if options.verbose:
+    logger.setLevel(logging.DEBUG)
+
+if options.full_testbase is not None:
+    testbase = options.full_testbase
+else:
+    testbase = "%s/b%u" % (options.testbase, os.getpid())
+test_master = "%s/master" % testbase
+test_prefix = "%s/prefix" % testbase
+test_tmpdir = "%s/tmp" % testbase
+os.environ['TMPDIR'] = test_tmpdir
+
+if options.enable_coverage:
+    LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
+else:
+    LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
+
+if options.enable_coverage:
+    PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
+else:
+    PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
+
+CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
+
+
+def check_symbols(sofile, expected_symbols=""):
+    return "objdump --dynamic-syms " + sofile + " | " + \
+           "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
+
+if args:
+    # If we are only running specific test,
+    # do not sleep randomly to wait for it to start
+    def random_sleep(low, high):
+        return 'sleep 1'
+else:
+    def random_sleep(low, high):
+        return 'sleep {}'.format(random.randint(low, high))
+
+cleanup_list = []
+
+builddirs = {
+    "ctdb": "ctdb",
+    "ldb": "lib/ldb",
+    "tdb": "lib/tdb",
+    "talloc": "lib/talloc",
+    "replace": "lib/replace",
+    "tevent": "lib/tevent",
+    "pidl": "pidl",
+    "docs-xml": "docs-xml"
+}
+
+ctdb_configure_params = " --enable-developer ${PREFIX}"
+samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
+
+samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
+samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
+samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
+samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
+samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
+samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
+samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
+
+
+def format_option(name, value=None):
+    """Format option as str list."""
+    if value is None:  # boolean option
+        return [name]
+    if not isinstance(value, list):  # single value option
+        value = [value]
+    # repeatable option
+    return ['{}={}'.format(name, item) for item in value]
+
+
+def make_test(
+        cmd='make testonly',
+        INJECT_SELFTEST_PREFIX=1,
+        TESTS='',
+        include_envs=None,
+        exclude_envs=None):
+
+    test_options = []
+    if include_envs:
+        test_options = format_option('--include-env', include_envs)
+    if exclude_envs:
+        test_options = format_option('--exclude-env', exclude_envs)
+    if test_options:
+        # join envs options to original test options
+        TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
+
+    _options = []
+
+    # Allow getting a full CI with
+    # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
+
+    FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
+
+    if int(FAIL_IMMEDIATELY):
+        _options.append('FAIL_IMMEDIATELY=1')
+    if TESTS:
+        _options.append("TESTS='{}'".format(TESTS))
+
+    if INJECT_SELFTEST_PREFIX:
+        _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
+        _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
+
+    return ' '.join([cmd] + _options)
+
+
+# When updating this list, also update .gitlab-ci.yml to add the job
+# and to make it a dependency of 'page' for the coverage report.
+
+tasks = {
+    "ctdb": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("configure", "./configure " + ctdb_configure_params),
+            ("make", "make all"),
+            ("install", "make install"),
+            ("test", "make autotest"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+        ],
+    },
+    "docs-xml": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("autoconf", "autoconf"),
+            ("configure", "./configure"),
+            ("make", "make html htmlman"),
+            ("publish-docs", PUBLISH_DOCS),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "samba-def-build": {
+        "git-clone-required": True,
+        "sequence": [
+            ("configure", "./configure.developer" + samba_configure_params),
+            ("make", "make -j"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("chmod-R-a-w", "chmod -R a-w ."),
+        ],
+    },
+
+    "samba-mit-build": {
+        "git-clone-required": True,
+        "sequence": [
+            ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
+            ("make", "make -j"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("chmod-R-a-w", "chmod -R a-w ."),
+        ],
+    },
+
+    "samba-nt4-build": {
+        "git-clone-required": True,
+        "sequence": [
+            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
+            ("make", "make -j"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("chmod-R-a-w", "chmod -R a-w ."),
+        ],
+    },
+
+    "samba-h5l-build": {
+        "git-clone-required": True,
+        "sequence": [
+            ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
+            ("make", "make -j"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("chmod-R-a-w", "chmod -R a-w ."),
+        ],
+    },
+
+    "samba-without-smb1-build": {
+        "git-clone-required": True,
+        "sequence": [
+            ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
+            ("make", "make -j"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("chmod-R-a-w", "chmod -R a-w ."),
+        ],
+    },
+
+    "samba-no-opath-build": {
+        "git-clone-required": True,
+        "sequence": [
+            ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1 -DDISABLE_VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS=1 -DDISABLE_PROC_FDS=1' ./configure.developer --without-ad-dc " + samba_configure_params),
+            ("make", "make -j"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("chmod-R-a-w", "chmod -R a-w ."),
+        ],
+    },
+
+    # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
+    "samba": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("configure", "./configure.developer" + samba_configure_params),
+            ("make", "make -j"),
+            ("test", make_test(exclude_envs=[
+            "none",
+            "nt4_dc",
+            "nt4_dc_smb1",
+            "nt4_dc_smb1_done",
+            "nt4_dc_schannel",
+            "nt4_member",
+            "ad_dc",
+            "ad_dc_smb1",
+            "ad_dc_smb1_done",
+            "ad_dc_backup",
+            "ad_dc_ntvfs",
+            "ad_dc_default",
+            "ad_dc_default_smb1",
+            "ad_dc_slowtests",
+            "ad_dc_no_nss",
+            "ad_dc_no_ntlm",
+            "fl2003dc",
+            "fl2008dc",
+            "fl2008r2dc",
+            "ad_member",
+            "ad_member_idmap_rid",
+            "admem_idmap_autorid",
+            "ad_member_idmap_ad",
+            "ad_member_rfc2307",
+            "ad_member_idmap_nss",
+            "ad_member_oneway",
+            "chgdcpass",
+            "vampire_2000_dc",
+            "fl2000dc",
+            "fileserver",
+            "fileserver_smb1",
+            "fileserver_smb1_done",
+            "maptoguest",
+            "simpleserver",
+            "backupfromdc",
+            "restoredc",
+            "renamedc",
+            "offlinebackupdc",
+            "labdc",
+            "preforkrestartdc",
+            "proclimitdc",
+            "promoted_dc",
+            "vampire_dc",
+            "rodc",
+            "ad_dc_default",
+            "ad_dc_default_smb1",
+            "ad_dc_default_smb1_done",
+            "ad_dc_slowtests",
+            "schema_pair_dc",
+            "schema_dc",
+            "clusteredmember",
+            "ad_dc_fips",
+            "ad_member_fips",
+            ])),
+            ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
+            ("lcov", LCOV_CMD),
+            ("install", "make install"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+        ],
+    },
+
+    # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
+    "samba-mitkrb5": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
+            ("make", "make -j"),
+            ("test", make_test(exclude_envs=[
+            "none",
+            "nt4_dc",
+            "nt4_dc_smb1",
+            "nt4_dc_smb1_done",
+            "nt4_dc_schannel",
+            "nt4_member",
+            "ad_dc",
+            "ad_dc_smb1",
+            "ad_dc_smb1_done",
+            "ad_dc_backup",
+            "ad_dc_ntvfs",
+            "ad_dc_default",
+            "ad_dc_default_smb1",
+            "ad_dc_default_smb1_done",
+            "ad_dc_slowtests",
+            "ad_dc_no_nss",
+            "ad_dc_no_ntlm",
+            "fl2003dc",
+            "fl2008dc",
+            "fl2008r2dc",
+            "ad_member",
+            "ad_member_idmap_rid",
+            "admem_idmap_autorid",
+            "ad_member_idmap_ad",
+            "ad_member_rfc2307",
+            "ad_member_idmap_nss",
+            "ad_member_oneway",
+            "chgdcpass",
+            "vampire_2000_dc",
+            "fl2000dc",
+            "fileserver",
+            "fileserver_smb1",
+            "fileserver_smb1_done",
+            "maptoguest",
+            "simpleserver",
+            "backupfromdc",
+            "restoredc",
+            "renamedc",
+            "offlinebackupdc",
+            "labdc",
+            "preforkrestartdc",
+            "proclimitdc",
+            "promoted_dc",
+            "vampire_dc",
+            "rodc",
+            "ad_dc_default",
+            "ad_dc_default_smb1",
+            "ad_dc_default_smb1_done",
+            "ad_dc_slowtests",
+            "schema_pair_dc",
+            "schema_dc",
+            "clusteredmember",
+            "ad_dc_fips",
+            "ad_member_fips",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("install", "make install"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "samba-nt4": {
+        "dependency": "samba-nt4-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(include_envs=[
+            "nt4_dc",
+            "nt4_dc_smb1",
+            "nt4_dc_smb1_done",
+            "nt4_dc_schannel",
+            "nt4_member",
+            "simpleserver",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-fileserver": {
+        "dependency": "samba-h5l-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(include_envs=[
+            "fileserver",
+            "fileserver_smb1",
+            "fileserver_smb1_done",
+            "maptoguest",
+            "ktest", # ktest is also tested in samba-ktest-mit samba
+                     # and samba-mitkrb5 but is tested here against
+                     # a system Heimdal
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-fileserver-without-smb1": {
+        "dependency": "samba-without-smb1-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(include_envs=["fileserver"])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    # This is a full build without the AD DC so we test the build with
+    # MIT Kerberos from the current system.  Runtime behaviour is
+    # confirmed via the ktest (static ccache and keytab) environment
+
+    # This environment also used to confirm we can still build with --with-libunwind
+    "samba-ktest-mit": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
+            ("make", "make -j"),
+            ("test", make_test(include_envs=[
+            "ktest", # ktest is also tested in fileserver, samba and
+                     # samba-mitkrb5 but is tested here against a
+                     # system MIT krb5
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-admem": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(include_envs=[
+            "ad_member",
+            "ad_member_idmap_rid",
+            "admem_idmap_autorid",
+            "ad_member_idmap_ad",
+            "ad_member_rfc2307",
+            "ad_member_idmap_nss",
+            "ad_member_offlogon",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-no-opath1": {
+        "dependency": "samba-no-opath-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(
+                cmd="make testonly DISABLE_OPATH=1",
+                include_envs=[
+                "nt4_dc",
+                "nt4_dc_smb1",
+                "nt4_dc_smb1_done",
+                "nt4_dc_schannel",
+                "nt4_member",
+                "simpleserver",
+                ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", "script/clean-source-tree.sh"),
+        ],
+    },
+
+    "samba-no-opath2": {
+        "dependency": "samba-no-opath-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(
+                cmd="make testonly DISABLE_OPATH=1",
+                include_envs=[
+                "fileserver",
+                "fileserver_smb1",
+                "fileserver_smb1_done",
+                ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", "script/clean-source-tree.sh"),
+        ],
+    },
+
+    "samba-ad-dc-1": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "ad_dc",
+            "ad_dc_smb1",
+            "ad_dc_smb1_done",
+            "ad_dc_no_nss",
+            "ad_dc_no_ntlm",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-ad-dc-2": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "vampire_dc",
+            "vampire_2000_dc",
+            "rodc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-ad-dc-3": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "promoted_dc",
+            "chgdcpass",
+            "preforkrestartdc",
+            "proclimitdc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-ad-dc-4a": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "fl2000dc",
+            "ad_member_oneway",
+            "fl2003dc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+    "samba-ad-dc-4b": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "fl2008dc",
+            "fl2008r2dc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-ad-dc-5": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-ad-dc-6": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-schemaupgrade": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
+    # This is currently the longest task, so we don't randomly delay it.
+    "samba-ad-dc-ntvfs": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=["ad_dc_ntvfs"])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    # Test fips compliance
+    "samba-fips": {
+        "dependency": "samba-mit-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
+            # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    # run the backup/restore testenvs separately as they're fairly standalone
+    # (and CI seems to max out at ~3 different DCs running at once)
+    "samba-ad-back1": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(include_envs=[
+            "backupfromdc",
+            "restoredc",
+            "renamedc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+    "samba-ad-back2": {
+        "dependency": "samba-def-build",
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("test", make_test(include_envs=[
+            "backupfromdc",
+            "offlinebackupdc",
+            "labdc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-admem-mit": {
+        "dependency": "samba-mit-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "ad_member",
+            "ad_member_idmap_rid",
+            "admem_idmap_autorid",
+            "ad_member_idmap_ad",
+            "ad_member_rfc2307",
+            "ad_member_idmap_nss",
+            "ad_member_offlogon",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-addc-mit-1": {
+        "dependency": "samba-mit-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "ad_dc",
+            "ad_dc_smb1",
+            "ad_dc_smb1_done",
+            "ad_dc_no_nss",
+            "ad_dc_no_ntlm",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-addc-mit-4a": {
+        "dependency": "samba-mit-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "fl2000dc",
+            "ad_member_oneway",
+            "fl2003dc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+    "samba-addc-mit-4b": {
+        "dependency": "samba-mit-build",
+        "sequence": [
+            ("random-sleep", random_sleep(1, 1)),
+            ("test", make_test(include_envs=[
+            "fl2008dc",
+            "fl2008r2dc",
+            ])),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+        ],
+    },
+
+    "samba-test-only": {
+        "sequence": [
+            ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
+            ("make", "make -j"),
+            ("test", make_test(TESTS="${TESTS}")),
+            ("lcov", LCOV_CMD),
+        ],
+    },
+
+    # Test cross-compile infrastructure
+    "samba-xc": {
+        "sequence": [
+            ("random-sleep", random_sleep(900, 1500)),
+            ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
+            ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
+            " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
+            ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
+            ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
+            " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
+            ("compare-results", "script/compare_cc_results.py "
+            "./bin/c4che/default{} "
+            "./bin-xe/c4che/default{} "
+            "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
+            ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
+            ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
+            " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
+            ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
+            " = \"'1234'\"".format(CACHE_SUFFIX)),
+            ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
+            ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
+            " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
+            " ; test $? -ne 0"),
+        ],
+    },
+
+    # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
+    "samba-o3": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
+            ("make", "make -j"),
+            ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
+            ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
+            ("lcov", LCOV_CMD),
+            ("install", "make install"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "samba-32bit": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("configure", "./configure.developer --abi-check-disable --disable-warnings-as-errors" + samba_configure_params),
+            ("make", "make -j"),
+            ("nonetest", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
+            ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
+            ("ktest", make_test(cmd='make test', include_envs=["ktest"])),
+            ("install", "make install"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "samba-ctdb": {
+        "sequence": [
+            ("random-sleep", random_sleep(900, 1500)),
+
+        # make sure we have tdb around:
+            ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
+            ("tdb-make", "cd lib/tdb && make"),
+            ("tdb-install", "cd lib/tdb && make install"),
+
+        # build samba with cluster support (also building ctdb):
+            ("samba-configure",
+         "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
+         "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
+         "./configure.developer ${PREFIX} "
+         "--with-selftest-prefix=./bin/ab "
+         "--with-cluster-support "
+         "--without-ad-dc "
+         "--bundled-libraries=!tdb"),
+            ("samba-make", "make"),
+            ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
+            ("samba-install", "make install"),
+            ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
+
+            ("test", make_test(
+                cmd='PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH make test',
+                INJECT_SELFTEST_PREFIX=0,
+                include_envs=["clusteredmember"])
+            ),
+
+        # clean up:
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+            ("ctdb-clean", "cd ./ctdb && make clean"),
+        ],
+    },
+
+    "samba-libs": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+            ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
+            ("talloc-make", "cd lib/talloc && make"),
+            ("talloc-install", "cd lib/talloc && make install"),
+
+            ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
+            ("tdb-make", "cd lib/tdb && make"),
+            ("tdb-install", "cd lib/tdb && make install"),
+
+            ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
+            ("tevent-make", "cd lib/tevent && make"),
+            ("tevent-install", "cd lib/tevent && make install"),
+
+            ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
+            ("ldb-make", "cd lib/ldb && make"),
+            ("ldb-install", "cd lib/ldb && make install"),
+
+            ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"),
+            ("nondevel-make", "make -j"),
+            ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
+            ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
+            ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
+            ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
+            ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
+            ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
+            ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
+            ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
+            ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
+            ("nondevel-no-public-nss_winbind",
+                check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
+            ("nondevel-no-public-nss_wins",
+                check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
+            ("nondevel-no-public-libwbclient",
+                check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
+            ("nondevel-no-public-pam_winbind",
+                check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
+            ("nondevel-no-public-winbind_krb5_locator",
+                check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
+            ("nondevel-no-public-async_dns_krb5_locator",
+                check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
+            ("nondevel-install", "make -j install"),
+            ("nondevel-dist", "make dist"),
+
+            ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
+            ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
+            ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
+            ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
+            ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
+            ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
+            ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
+            ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
+            ("prefix-no-public-nss_winbind",
+                check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
+            ("prefix-no-public-nss_wins",
+                check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
+            ("prefix-no-public-libwbclient",
+                check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
+            ("prefix-no-public-pam_winbind",
+                check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
+            ("prefix-no-public-winbind_krb5_locator",
+                check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
+                              "service_locator")),
+            ("prefix-no-public-async_dns_krb5_locator",
+                check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
+                              "service_locator")),
+
+            # retry with all modules shared
+            ("allshared-distclean", "make distclean"),
+            ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
+            ("allshared-make", "make -j"),
+            ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
+            ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
+            ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
+            ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
+            ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
+            ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
+            ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
+            ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
+            ("allshared-no-public-nss_winbind",
+                check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
+            ("allshared-no-public-nss_wins",
+                check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
+            ("allshared-no-public-libwbclient",
+                check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
+            ("allshared-no-public-pam_winbind",
+                check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
+            ("allshared-no-public-winbind_krb5_locator",
+                check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
+            ("allshared-no-public-async_dns_krb5_locator",
+                check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
+        ],
+    },
+
+    "samba-fuzz": {
+        "sequence": [
+        # build the fuzzers (static) via the oss-fuzz script
+            ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
+            ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
+        ],
+    },
+
+    # * Test smbd and smbtorture can build semi-static
+    #
+    # * Test Samba without python still builds.
+    #
+    # When this test fails due to more use of Python, the expectations
+    # is that the newly failing part of the code should be disabled
+    # when --disable-python is set (rather than major work being done
+    # to support this environment).
+    #
+    # The target here is for vendors shipping a minimal smbd.
+    "samba-minimal-smbd": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+
+        # build with all modules static
+            ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
+            ("allstatic-make", "make -j"),
+            ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
+            ("allstatic-lcov", LCOV_CMD),
+            ("allstatic-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("allstatic-def-clean", "make clean"),
+
+        # force all libraries as private
+            ("allprivate-def-distclean", "make distclean"),
+            ("allprivate-def-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL"),
+            ("allprivate-def-make", "make -j"),
+            # note wrapper libraries need to be public
+            ("allprivate-def-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so' | wc -l | grep -q '^0'"),
+            ("allprivate-def-only-private-ext", "ls ./bin/shared/private | egrep 'private-samba' | wc -l | grep -q '^0' && exit 1; exit 0"),
+            ("allprivate-def-no-non-private-ext", "ls ./bin/shared/private | egrep -v 'private-samba|^libpypamtest.so$' | wc -l | grep -q '^0'"),
+            ("allprivate-def-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
+            ("allprivate-def-lcov", LCOV_CMD),
+            ("allprivate-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("allprivate-def-clean", "make clean"),
+
+        # force all libraries as private with a non default
+        # extension and 2 exceptions
+            ("allprivate-ext-distclean", "make distclean"),
+            ("allprivate-ext-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL --private-library-extension=private-library --private-extension-exception=pac,ndr"),
+            ("allprivate-ext-make", "make -j"),
+            # note wrapper libraries need to be public
+            ("allprivate-ext-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so' | wc -l | grep -q '^0'"),
+            ("allprivate-ext-no-private-default-ext", "ls ./bin/shared/private | grep 'private-samba' | wc -l | grep -q '^0'"),
+            ("allprivate-ext-has-private-ext", "ls ./bin/shared/private | grep 'private-library' | wc -l | grep -q '^0' && exit 1; exit 0"),
+            ("allprivate-ext-libndr-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libndr' | wc -l | grep -q '^1'"),
+            ("allprivate-ext-libpac-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libpac' | wc -l | grep -q '^1'"),
+            ("allprivate-ext-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
+            ("allprivate-ext-lcov", LCOV_CMD),
+            ("allprivate-ext-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("allprivate-ext-clean", "make clean"),
+
+        # retry with nonshared smbd and smbtorture
+            ("nonshared-distclean", "make distclean"),
+            ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
+            ("nonshared-make", "make -j"),
+            ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
+            ("nonshared-lcov", LCOV_CMD),
+            ("nonshared-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("nonshared-clean", "make clean"),
+        ],
+    },
+
+    "samba-nopython": {
+        "sequence": [
+            ("random-sleep", random_sleep(300, 900)),
+
+            ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
+            ("make", "make -j"),
+            ("find-python", "script/find_python.sh ${PREFIX}"),
+            ("test", "make test-nopython"),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+
+            ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
+            ("talloc-make", "cd lib/talloc && make"),
+            ("talloc-install", "cd lib/talloc && make install"),
+
+            ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
+            ("tdb-make", "cd lib/tdb && make"),
+            ("tdb-install", "cd lib/tdb && make install"),
+
+            ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
+            ("tevent-make", "cd lib/tevent && make"),
+            ("tevent-install", "cd lib/tevent && make install"),
+
+            ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
+            ("ldb-make", "cd lib/ldb && make"),
+            ("ldb-install", "cd lib/ldb && make install"),
+
+        # retry against installed library packages, but no required modules
+            ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
+            ("libs-make", "make -j"),
+            ("libs-install", "make install"),
+            ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("libs-clean", "make clean"),
+
+        ],
+    },
+
+    "samba-codecheck": {
+        "sequence": [
+            ("run", "script/check-shell-scripts.sh ."),
+            ("run", "script/codespell.sh ."),
+        ],
+    },
+
+    "ldb": {
+        "sequence": [
+            ("random-sleep", random_sleep(60, 600)),
+            ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+            ("make", "make"),
+            ("install", "make install"),
+            ("test", "make test"),
+            ("lcov", LCOV_CMD),
+            ("clean", "make clean"),
+            ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
+            ("make-no-lmdb", "make"),
+            ("test-no-lmdb", "make test"),
+            ("lcov-no-lmdb", LCOV_CMD),
+            ("install-no-lmdb", "make install"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("distcheck", "make distcheck"),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "tdb": {
+        "sequence": [
+            ("random-sleep", random_sleep(60, 600)),
+            ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+            ("make", "make"),
+            ("install", "make install"),
+            ("test", "make test"),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("distcheck", "make distcheck"),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "talloc": {
+        "sequence": [
+            ("random-sleep", random_sleep(60, 600)),
+            ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+            ("make", "make"),
+            ("install", "make install"),
+            ("test", "make test"),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("distcheck", "make distcheck"),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "replace": {
+        "sequence": [
+            ("random-sleep", random_sleep(60, 600)),
+            ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+            ("make", "make"),
+            ("install", "make install"),
+            ("test", "make test"),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("distcheck", "make distcheck"),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "tevent": {
+        "sequence": [
+            ("random-sleep", random_sleep(60, 600)),
+            ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
+            ("make", "make"),
+            ("install", "make install"),
+            ("test", "make test"),
+            ("lcov", LCOV_CMD),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("distcheck", "make distcheck"),
+            ("clean", "make clean"),
+        ],
+    },
+
+    "pidl": {
+        "git-clone-required": True,
+        "sequence": [
+            ("random-sleep", random_sleep(60, 600)),
+            ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
+            ("touch", "touch *.yp"),
+            ("make", "make"),
+            ("test", "make test"),
+            ("install", "make install"),
+            ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
+            ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
+            ("clean", "make clean"),
+        ],
+    },
+
+    # these are useful for debugging autobuild
+    "pass": {
+        "sequence": [
+            ("pass", 'echo passing && /bin/true'),
+        ],
+    },
+    "fail": {
+        "sequence": [
+            ("fail", 'echo failing && /bin/false'),
+        ],
+    },
+}
+
+defaulttasks = list(tasks.keys())
+
+defaulttasks.remove("pass")
+defaulttasks.remove("fail")
+
+# The build tasks will be brought in by the test tasks as needed
+defaulttasks.remove("samba-def-build")
+defaulttasks.remove("samba-nt4-build")
+defaulttasks.remove("samba-mit-build")
+defaulttasks.remove("samba-h5l-build")
+defaulttasks.remove("samba-no-opath-build")
+
+# This is not a normal test, but a task to support manually running
+# one test under autobuild
+defaulttasks.remove("samba-test-only")
+
+# Only built on GitLab CI and not in the default autobuild because it
+# uses too much space (4GB of semi-static binaries)
+defaulttasks.remove("samba-fuzz")
+
+# The FIPS build runs only in GitLab CI on a current Fedora Docker
+# container where a simulated FIPS mode is possible.
+defaulttasks.remove("samba-fips")
+
+# The MIT build runs on a current Fedora where an up to date MIT KDC
+# is already packaged.  This avoids needing to backport a current MIT
+# to the default Ubuntu 18.04, particularly during development, and
+# the need to install on the shared sn-devel-184.
+
+defaulttasks.remove("samba-mitkrb5")
+defaulttasks.remove("samba-admem-mit")
+defaulttasks.remove("samba-addc-mit-1")
+defaulttasks.remove("samba-addc-mit-4a")
+defaulttasks.remove("samba-addc-mit-4b")
+
+defaulttasks.remove("samba-32bit")
+
+if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
+    defaulttasks.remove("samba-o3")
+
+
+def do_print(msg):
+    logger.info(msg)
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+def do_debug(msg):
+    logger.debug(msg)
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+
+def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
+    if show is None:
+        do_debug("Running: '%s' in '%s'" % (cmd, dir))
+    elif show:
+        do_print("Running: '%s' in '%s'" % (cmd, dir))
+
+    if output:
+        out = check_output([cmd], shell=True, cwd=dir)
+        return out.decode(encoding='utf-8', errors='backslashreplace')
+    elif checkfail:
+        return check_call(cmd, shell=True, cwd=dir)
+    else:
+        return call(cmd, shell=True, cwd=dir)
+
+def rmdir_force(dirname, re_raise=True):
+    try:
+        run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
+                dirname, dirname, dirname), output=True, show=True)
+    except CalledProcessError as e:
+        do_print("Failed: '%s'" % (str(e)))
+        run_cmd("tree %s" % dirname, output=True, show=True)
+        if re_raise:
+            raise
+        return False
+    return True
+
+class builder(object):
+    '''handle build of one directory'''
+
+    def __init__(self, name, definition):
+        self.name = name
+        self.dir = builddirs.get(name, '.')
+        self.tag = self.name.replace('/', '_')
+        self.definition = definition
+        self.sequence = definition["sequence"]
+        self.git_clone_required = False
+        if "git-clone-required" in definition:
+            self.git_clone_required = bool(definition["git-clone-required"])
+        self.proc = None
+        self.done = False
+        self.next = 0
+        self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
+        self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
+        do_debug("stdout for %s in %s" % (self.name, self.stdout_path))
+        do_debug("stderr for %s in %s" % (self.name, self.stderr_path))
+        run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
+        self.stdout = open(self.stdout_path, 'w')
+        self.stderr = open(self.stderr_path, 'w')
+        self.stdin  = open("/dev/null", 'r')
+        self.builder_dir = "%s/%s" % (testbase, self.tag)
+        self.test_source_dir = self.builder_dir
+        self.cwd = "%s/%s" % (self.builder_dir, self.dir)
+        self.selftest_prefix = "%s/bin/ab" % (self.cwd)
+        self.prefix = "%s/%s" % (test_prefix, self.tag)
+        self.consumers = []
+        self.producer = None
+
+        if self.git_clone_required:
+            assert "dependency" not in definition
+
+    def mark_existing(self):
+        do_debug('%s: Mark as existing dependency' % self.name)
+        self.next = len(self.sequence)
+        self.done = True
+
+    def add_consumer(self, consumer):
+        do_debug("%s: add consumer: %s" % (self.name, consumer.name))
+        consumer.producer = self
+        consumer.test_source_dir = self.test_source_dir
+        self.consumers.append(consumer)
+
+    def start_next(self):
+        if self.producer is not None:
+            if not self.producer.done:
+                do_debug("%s: Waiting for producer: %s" % (self.name, self.producer.name))
+                return
+
+        if self.next == 0:
+            rmdir_force(self.builder_dir)
+            rmdir_force(self.prefix)
+            if self.producer is not None:
+                run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
+            elif not self.git_clone_required:
+                run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
+            else:
+                run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
+
+        if self.next == len(self.sequence):
+            if not self.done:
+                do_print('%s: Completed OK' % self.name)
+                self.done = True
+            if not options.nocleanup and len(self.consumers) == 0:
+                do_print('%s: Cleaning up' % self.name)
+                rmdir_force(self.builder_dir)
+                rmdir_force(self.prefix)
+            for consumer in self.consumers:
+                if consumer.next != 0:
+                    continue
+                do_print('%s: Starting consumer %s' % (self.name, consumer.name))
+                consumer.start_next()
+            if self.producer is not None:
+                self.producer.consumers.remove(self)
+                assert self.producer.done
+                self.producer.start_next()
+            do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
+            return
+        (self.stage, self.cmd) = self.sequence[self.next]
+        self.cmd = self.cmd.replace("${PYTHON_PREFIX}",
+                                    get_path(name='platlib',
+                                             scheme="posix_prefix",
+                                             vars={"base": self.prefix,
+                                                   "platbase": self.prefix}))
+        self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
+        self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
+        self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
+        self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
+        self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
+        self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
+        self.cmd = self.cmd.replace("${NAME}", self.name)
+        self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
+        do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
+        self.proc = Popen(self.cmd, shell=True,
+                          close_fds=True, cwd=self.cwd,
+                          stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
+        self.next += 1
+
+def expand_dependencies(n):
+    deps = list()
+    if "dependency" in tasks[n]:
+        depname = tasks[n]["dependency"]
+        assert depname in tasks
+        sdeps = expand_dependencies(depname)
+        assert n not in sdeps
+        for sdep in sdeps:
+            deps.append(sdep)
+        deps.append(depname)
+    return deps
+
+
+class buildlist(object):
+    '''handle build of multiple directories'''
+
+    def __init__(self, tasknames, rebase_url, rebase_branch="master"):
+        self.tail_proc = None
+        self.retry = None
+        if not tasknames:
+            if options.restrict_tests:
+                tasknames = ["samba-test-only"]
+            else:
+                tasknames = defaulttasks
+
+        given_tasknames = tasknames.copy()
+        implicit_tasknames = []
+        for n in given_tasknames:
+            deps = expand_dependencies(n)
+            for dep in deps:
+                if dep in given_tasknames:
+                    continue
+                if dep in implicit_tasknames:
+                    continue
+                implicit_tasknames.append(dep)
+
+        tasknames = implicit_tasknames.copy()
+        tasknames.extend(given_tasknames)
+        do_debug("given_tasknames: %s" % given_tasknames)
+        do_debug("implicit_tasknames: %s" % implicit_tasknames)
+        do_debug("tasknames: %s" % tasknames)
+        self.tlist = [builder(n, tasks[n]) for n in tasknames]
+
+        if options.retry:
+            rebase_remote = "rebaseon"
+            retry_task = {
+                    "git-clone-required": True,
+                    "sequence": [
+                            ("retry",
+                            '''set -e
+                            git remote add -t %s %s %s
+                            git fetch %s
+                            while :; do
+                              sleep 60
+                              git describe %s/%s > old_remote_branch.desc
+                              git fetch %s
+                              git describe %s/%s > remote_branch.desc
+                              diff old_remote_branch.desc remote_branch.desc
+                            done
+                           ''' % (
+                               rebase_branch, rebase_remote, rebase_url,
+                               rebase_remote,
+                               rebase_remote, rebase_branch,
+                               rebase_remote,
+                               rebase_remote, rebase_branch
+                            ))]}
+
+            self.retry = builder('retry', retry_task)
+            self.need_retry = False
+
+        if options.skip_dependencies:
+            for b in self.tlist:
+                if b.name in implicit_tasknames:
+                    b.mark_existing()
+
+        for b in self.tlist:
+            do_debug("b.name=%s" % b.name)
+            if "dependency" not in b.definition:
+                continue
+            depname = b.definition["dependency"]
+            do_debug("b.name=%s: dependency:%s" % (b.name, depname))
+            for p in self.tlist:
+                if p.name == depname:
+                    p.add_consumer(b)
+
+    def kill_kids(self):
+        if self.tail_proc is not None:
+            self.tail_proc.terminate()
+            self.tail_proc.wait()
+            self.tail_proc = None
+        if self.retry is not None:
+            self.retry.proc.terminate()
+            self.retry.proc.wait()
+            self.retry = None
+        for b in self.tlist:
+            if b.proc is not None:
+                run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
+                b.proc.terminate()
+                b.proc.wait()
+                b.proc = None
+
+    def wait_one(self):
+        while True:
+            none_running = True
+            for b in self.tlist:
+                if b.proc is None:
+                    continue
+                none_running = False
+                b.status = b.proc.poll()
+                if b.status is None:
+                    continue
+                b.proc = None
+                return b
+            if options.retry:
+                ret = self.retry.proc.poll()
+                if ret is not None:
+                    self.need_retry = True
+                    self.retry = None
+                    return None
+            if none_running:
+                return None
+            time.sleep(0.1)
+
+    def run(self):
+        for b in self.tlist:
+            b.start_next()
+        if options.retry:
+            self.retry.start_next()
+        while True:
+            b = self.wait_one()
+            if options.retry and self.need_retry:
+                self.kill_kids()
+                do_print("retry needed")
+                return (0, None, None, None, "retry")
+            if b is None:
+                break
+            if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
+                self.kill_kids()
+                return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
+            b.start_next()
+        self.kill_kids()
+        return (0, None, None, None, "All OK")
+
+    def write_system_info(self, filename):
+        with open(filename, 'w') as f:
+            for cmd in ['uname -a',
+                        'lsb_release -a',
+                        'free',
+                        'mount',
+                        'cat /proc/cpuinfo',
+                        'cc --version',
+                        'df -m .',
+                        'df -m %s' % testbase]:
+                try:
+                    out = run_cmd(cmd, output=True, checkfail=False)
+                except CalledProcessError as e:
+                    out = "<failed: %s>" % str(e)
+                print('### %s' % cmd, file=f)
+                print(out, file=f)
+                print(file=f)
+
+    def tarlogs(self, fname):
+        with tarfile.open(fname, "w:gz") as tar:
+            for b in self.tlist:
+                tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
+                tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
+            if os.path.exists("autobuild.log"):
+                tar.add("autobuild.log")
+            filename = 'system-info.txt'
+            self.write_system_info(filename)
+            tar.add(filename)
+
+    def remove_logs(self):
+        for b in self.tlist:
+            os.unlink(b.stdout_path)
+            os.unlink(b.stderr_path)
+
+    def start_tail(self):
+        cmd = ["tail", "-f"]
+        for b in self.tlist:
+            cmd.append(b.stdout_path)
+            cmd.append(b.stderr_path)
+        self.tail_proc = Popen(cmd, close_fds=True)
+
+
+def cleanup(do_raise=False):
+    if options.nocleanup:
+        return
+    run_cmd("stat %s || true" % test_tmpdir, show=True)
+    run_cmd("stat %s" % testbase, show=True)
+    do_print("Cleaning up %r" % cleanup_list)
+    for d in cleanup_list:
+        ok = rmdir_force(d, re_raise=False)
+        if ok:
+            continue
+        if os.path.isdir(d):
+            do_print("Killing, waiting and retry")
+            run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
+        else:
+            do_print("Waiting and retry")
+        time.sleep(1)
+        rmdir_force(d, re_raise=do_raise)
+
+
+def daemonize(logfile):
+    pid = os.fork()
+    if pid == 0:  # Parent
+        os.setsid()
+        pid = os.fork()
+        if pid != 0:  # Actual daemon
+            os._exit(0)
+    else:  # Grandparent
+        os._exit(0)
+
+    import resource      # Resource usage information.
+    maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+    if maxfd == resource.RLIM_INFINITY:
+        maxfd = 1024  # Rough guess at maximum number of open file descriptors.
+    for fd in range(0, maxfd):
+        try:
+            os.close(fd)
+        except OSError:
+            pass
+    os.open(logfile, os.O_RDWR | os.O_CREAT)
+    os.dup2(0, 1)
+    os.dup2(0, 2)
+
+
+def write_pidfile(fname):
+    '''write a pid file, cleanup on exit'''
+    with open(fname, mode='w') as f:
+        f.write("%u\n" % os.getpid())
+
+
+def rebase_tree(rebase_url, rebase_branch="master"):
+    rebase_remote = "rebaseon"
+    do_print("Rebasing on %s" % rebase_url)
+    run_cmd("git describe HEAD", show=True, dir=test_master)
+    run_cmd("git remote add -t %s %s %s" %
+            (rebase_branch, rebase_remote, rebase_url),
+            show=True, dir=test_master)
+    run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
+    if options.fix_whitespace:
+        run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
+                (rebase_remote, rebase_branch),
+                show=True, dir=test_master)
+    else:
+        run_cmd("git rebase --force-rebase %s/%s" %
+                (rebase_remote, rebase_branch),
+                show=True, dir=test_master)
+    diff = run_cmd("git --no-pager diff HEAD %s/%s" %
+                   (rebase_remote, rebase_branch),
+                   dir=test_master, output=True)
+    if diff == '':
+        do_print("No differences between HEAD and %s/%s - exiting" %
+                 (rebase_remote, rebase_branch))
+        sys.exit(0)
+    run_cmd("git describe %s/%s" %
+            (rebase_remote, rebase_branch),
+            show=True, dir=test_master)
+    run_cmd("git describe HEAD", show=True, dir=test_master)
+    run_cmd("git --no-pager diff --stat HEAD %s/%s" %
+            (rebase_remote, rebase_branch),
+            show=True, dir=test_master)
+
+
+def push_to(push_url, push_branch="master"):
+    push_remote = "pushto"
+    do_print("Pushing to %s" % push_url)
+    if options.mark:
+        run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
+        run_cmd("git commit --amend -c HEAD", dir=test_master)
+        # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
+        # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
+    run_cmd("git remote add -t %s %s %s" %
+            (push_branch, push_remote, push_url),
+            show=True, dir=test_master)
+    run_cmd("git push %s +HEAD:%s" %
+            (push_remote, push_branch),
+            show=True, dir=test_master)
+
+
+def send_email(subject, text, log_tar):
+    if options.email is None:
+        do_print("not sending email because the recipient is not set")
+        do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
+                 (subject, text))
+        return
+    outer = MIMEMultipart()
+    outer['Subject'] = subject
+    outer['To'] = options.email
+    outer['From'] = options.email_from
+    outer['Date'] = email.utils.formatdate(localtime=True)
+    outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
+    outer.attach(MIMEText(text, 'plain', 'utf-8'))
+    if options.attach_logs:
+        with open(log_tar, 'rb') as fp:
+            msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
+        # Set the filename parameter
+        msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
+        outer.attach(msg)
+    content = outer.as_string()
+    s = smtplib.SMTP(options.email_server)
+    email_user = os.getenv('SMTP_USERNAME')
+    email_password = os.getenv('SMTP_PASSWORD')
+    if email_user is not None:
+        s.starttls()
+        s.login(email_user, email_password)
+
+    s.sendmail(options.email_from, [options.email], content)
+    s.set_debuglevel(1)
+    s.quit()
+
+
+def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
+                  elapsed_time, log_base=None, add_log_tail=True):
+    '''send an email to options.email about the failure'''
+    elapsed_minutes = elapsed_time / 60.0
+    if log_base is None:
+        log_base = gitroot
+    text = '''
+Dear Developer,
+
+Your autobuild on %s failed after %.1f minutes
+when trying to test %s with the following error:
+
+   %s
+
+the autobuild has been abandoned. Please fix the error and resubmit.
+
+A summary of the autobuild process is here:
+
+  %s/autobuild.log
+''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
+
+    if options.restrict_tests:
+        text += """
+The build was restricted to tests matching %s\n""" % options.restrict_tests
+
+    if failed_task != 'rebase':
+        text += '''
+You can see logs of the failed task here:
+
+  %s/%s.stdout
+  %s/%s.stderr
+
+or you can get full logs of all tasks in this job here:
+
+  %s/logs.tar.gz
+
+The top commit for the tree that was built was:
+
+%s
+
+''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
+
+    log_stdout = "%s/%s.stdout" % (gitroot, failed_tag)
+    if add_log_tail and os.access(log_stdout, os.R_OK):
+        f = open(log_stdout, 'r')
+        lines = f.readlines()
+        log_tail = "".join(lines[-50:])
+        num_lines = len(lines)
+        log_stderr = "%s/%s.stderr" % (gitroot, failed_tag)
+        if num_lines < 50 and os.access(log_stderr, os.R_OK):
+            # Also include stderr (compile failures) if < 50 lines of stdout
+            f = open(log_stderr, 'r')
+            log_tail += "".join(f.readlines()[-(50 - num_lines):])
+
+        text += '''
+The last 50 lines of log messages:
+
+%s
+    ''' % log_tail
+        f.close()
+
+    logs = os.path.join(gitroot, 'logs.tar.gz')
+    send_email('autobuild[%s] failure on %s for task %s during %s'
+               % (options.branch, platform.node(), failed_task, failed_stage),
+               text, logs)
+
+
+def email_success(elapsed_time, log_base=None):
+    '''send an email to options.email about a successful build'''
+    if log_base is None:
+        log_base = gitroot
+    text = '''
+Dear Developer,
+
+Your autobuild on %s has succeeded after %.1f minutes.
+
+''' % (platform.node(), elapsed_time / 60.)
+
+    if options.restrict_tests:
+        text += """
+The build was restricted to tests matching %s\n""" % options.restrict_tests
+
+    if options.keeplogs:
+        text += '''
+
+you can get full logs of all tasks in this job here:
+
+  %s/logs.tar.gz
+
+''' % log_base
+
+    text += '''
+The top commit for the tree that was built was:
+
+%s
+''' % top_commit_msg
+
+    logs = os.path.join(gitroot, 'logs.tar.gz')
+    send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
+               text, logs)
+
+
+# get the top commit message, for emails
+top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
+
+try:
+    if options.skip_dependencies:
+        run_cmd("stat %s" % testbase, dir=testbase, output=True)
+    else:
+        os.makedirs(testbase)
+except Exception as reason:
+    raise Exception("Unable to create %s : %s" % (testbase, reason))
+cleanup_list.append(testbase)
+
+if options.daemon:
+    logfile = os.path.join(testbase, "log")
+    do_print("Forking into the background, writing progress to %s" % logfile)
+    daemonize(logfile)
+
+write_pidfile(gitroot + "/autobuild.pid")
+
+start_time = time.time()
+
+while True:
+    try:
+        run_cmd("rm -rf %s" % test_tmpdir, show=True)
+        os.makedirs(test_tmpdir)
+        # The waf uninstall code removes empty directories all the way
+        # up the tree.  Creating a file in test_tmpdir stops it from
+        # being removed.
+        run_cmd("touch %s" % os.path.join(test_tmpdir,
+                                          ".directory-is-not-empty"), show=True)
+        run_cmd("stat %s" % test_tmpdir, show=True)
+        run_cmd("stat %s" % testbase, show=True)
+        if options.skip_dependencies:
+            run_cmd("stat %s" % test_master, dir=testbase, output=True)
+        else:
+            run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
+    except Exception:
+        cleanup()
+        raise
+
+    try:
+        if options.rebase is not None:
+            rebase_tree(options.rebase, rebase_branch=options.branch)
+    except Exception:
+        cleanup_list.append(gitroot + "/autobuild.pid")
+        cleanup()
+        elapsed_time = time.time() - start_time
+        email_failure(-1, 'rebase', 'rebase', 'rebase',
+                      'rebase on %s failed' % options.branch,
+                      elapsed_time, log_base=options.log_base)
+        sys.exit(1)
+
+    try:
+        blist = buildlist(args, options.rebase, rebase_branch=options.branch)
+        if options.tail:
+            blist.start_tail()
+        (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
+        if status != 0 or errstr != "retry":
+            break
+        cleanup(do_raise=True)
+    except Exception:
+        cleanup()
+        raise
+
+cleanup_list.append(gitroot + "/autobuild.pid")
+
+do_print(errstr)
+
+blist.kill_kids()
+if options.tail:
+    do_print("waiting for tail to flush")
+    time.sleep(1)
+
+elapsed_time = time.time() - start_time
+if status == 0:
+    if options.passcmd is not None:
+        do_print("Running passcmd: %s" % options.passcmd)
+        run_cmd(options.passcmd, dir=test_master)
+    if options.pushto is not None:
+        push_to(options.pushto, push_branch=options.branch)
+    if options.keeplogs or options.attach_logs:
+        blist.tarlogs("logs.tar.gz")
+        do_print("Logs in logs.tar.gz")
+    if options.always_email:
+        email_success(elapsed_time, log_base=options.log_base)
+    blist.remove_logs()
+    cleanup()
+    do_print(errstr)
+    sys.exit(0)
+
+# something failed, gather a tar of the logs
+blist.tarlogs("logs.tar.gz")
+
+if options.email is not None:
+    email_failure(status, failed_task, failed_stage, failed_tag, errstr,
+                  elapsed_time, log_base=options.log_base)
+else:
+    elapsed_minutes = elapsed_time / 60.0
+    print('''
+
+####################################################################
+
+AUTOBUILD FAILURE
+
+Your autobuild[%s] on %s failed after %.1f minutes
+when trying to test %s with the following error:
+
+   %s
+
+the autobuild has been abandoned. Please fix the error and resubmit.
+
+####################################################################
+
+''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
+
+cleanup()
+do_print(errstr)
+do_print("Logs in logs.tar.gz")
+sys.exit(status)
diff --git a/script/bisect-test.py b/script/bisect-test.py
new file mode 100755
index 0000000..7c5cd63
--- /dev/null
+++ b/script/bisect-test.py
@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+# use git bisect to work out what commit caused a test failure
+# Copyright Andrew Tridgell 2010
+# released under GNU GPL v3 or later
+
+
+from subprocess import call, check_call, Popen, PIPE
+import os
+import tempfile
+import sys
+from optparse import OptionParser
+
+parser = OptionParser()
+parser.add_option("", "--good", help="known good revision (default HEAD~100)", default='HEAD~100')
+parser.add_option("", "--bad", help="known bad revision (default HEAD)", default='HEAD')
+parser.add_option("", "--skip-build-errors", help="skip revision where make fails",
+                  action='store_true', default=False)
+parser.add_option("", "--autogen", help="run autogen before each build", action="store_true", default=False)
+parser.add_option("", "--autogen-command", help="command to use for autogen (default ./autogen.sh)",
+                  type='str', default="./autogen.sh")
+parser.add_option("", "--configure", help="run configure.developer before each build",
+                  action="store_true", default=False)
+parser.add_option("", "--configure-command", help="the command for configure (default ./configure.developer)",
+                  type='str', default="./configure.developer")
+parser.add_option("", "--build-command", help="the command to build the tree (default 'make -j')",
+                  type='str', default="make -j")
+parser.add_option("", "--test-command", help="the command to test the tree (default 'make test')",
+                  type='str', default="make test")
+parser.add_option("", "--clean", help="run make clean before each build",
+                  action="store_true", default=False)
+
+
+(opts, args) = parser.parse_args()
+
+
+def run_cmd(cmd, dir=".", show=True, output=False, checkfail=True):
+    if show:
+        print("Running: '%s' in '%s'" % (cmd, dir))
+    if output:
+        return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
+    elif checkfail:
+        return check_call(cmd, shell=True, cwd=dir)
+    else:
+        return call(cmd, shell=True, cwd=dir)
+
+
+def find_git_root():
+    '''get to the top of the git repo'''
+    p = os.getcwd()
+    while p != '/':
+        if os.path.exists(os.path.join(p, ".git")):
+            return p
+        p = os.path.abspath(os.path.join(p, '..'))
+    return None
+
+
+cwd = os.getcwd()
+gitroot = find_git_root()
+
+# create a bisect script
+f = tempfile.NamedTemporaryFile(delete=False, mode="w+t")
+f.write("set -x\n")
+f.write("cd %s || exit 125\n" % cwd)
+if opts.autogen:
+    f.write("%s || exit 125\n" % opts.autogen_command)
+if opts.configure:
+    f.write("%s || exit 125\n" % opts.configure_command)
+if opts.clean:
+    f.write("make clean || exit 125\n")
+if opts.skip_build_errors:
+    build_err = 125
+else:
+    build_err = 1
+f.write("%s || exit %u\n" % (opts.build_command, build_err))
+f.write("%s || exit 1\n" % opts.test_command)
+f.write("exit 0\n")
+f.close()
+
+
+def cleanup():
+    run_cmd("git bisect reset", dir=gitroot)
+    os.unlink(f.name)
+    sys.exit(-1)
+
+
+# run bisect
+ret = -1
+try:
+    run_cmd("git bisect reset", dir=gitroot, show=False, checkfail=False)
+    run_cmd("git bisect start %s %s --" % (opts.bad, opts.good), dir=gitroot)
+    ret = run_cmd("git bisect run bash %s" % f.name, dir=gitroot, show=True, checkfail=False)
+except KeyboardInterrupt:
+    print("Cleaning up")
+    cleanup()
+except Exception as reason:
+    print("Failed bisect: %s" % reason)
+    cleanup()
+
+run_cmd("git bisect reset", dir=gitroot)
+os.unlink(f.name)
+sys.exit(ret)
diff --git a/script/check-shell-scripts.sh b/script/check-shell-scripts.sh
new file mode 100755
index 0000000..f4f48a3
--- /dev/null
+++ b/script/check-shell-scripts.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+if [ $# -lt 1 ]; then
+	echo "Usage: $(basename "${0}") DIR [SEVERITY]"
+	exit 1
+fi
+
+DIR="${1}"
+SEVERITY="${2:-error}"
+
+shfmt -f "${DIR}" |
+	grep -v -E "(bootstrap|third_party)" |
+	xargs shellcheck \
+		--shell=sh \
+		--external-sources \
+		--check-sourced \
+		--format=gcc \
+		--severity="${SEVERITY}"
+
+exit $?
diff --git a/script/clean-source-tree.sh b/script/clean-source-tree.sh
new file mode 100755
index 0000000..8d7f759
--- /dev/null
+++ b/script/clean-source-tree.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+
+N=$(git clean -n | wc -l)
+C=$(git diff --stat HEAD | wc -l)
+
+test x"$N" != x"0" && {
+	echo "The tree has $N new uncommitted files!!! see stderr"
+	echo "The tree has $N new uncommitted files!!!" >&2
+
+	echo "git clean -n" >&2
+	git clean -n >&2
+
+	test x"$C" != x"0" && {
+		echo "git diff -p --stat HEAD" >&2
+		git diff -p --stat HEAD >&2
+	}
+
+	exit 1
+}
+
+test x"$C" != x"0" && {
+	echo "The tree has uncommitted changes!!! see stderr"
+	echo "The tree has uncommitted changes!!!" >&2
+
+	echo "git diff -p --stat HEAD" >&2
+	git diff -p --stat HEAD >&2
+
+	exit 1
+}
+
+echo "clean tree"
+exit 0
diff --git a/script/codespell.sh b/script/codespell.sh
new file mode 100755
index 0000000..60e0eba
--- /dev/null
+++ b/script/codespell.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Check code spelling
+
+if [ $# -lt 1 ]; then
+	echo "Usage: $(basename "${0}") DIR"
+	exit 1
+fi
+
+DIR="${1}"
+
+codespell "${DIR}"
+ret=$?
+
+if [ ${ret} -ne 0 ]; then
+	echo
+	echo "Fix code spelling issues. If it detected false positives" \
+	     "please update .codespellignore."
+fi
+
+exit ${ret}
diff --git a/script/commit_mark.sh b/script/commit_mark.sh
new file mode 100755
index 0000000..3de6ba7
--- /dev/null
+++ b/script/commit_mark.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+# add a autobuild message to the HEAD commit
+
+branch=$(git branch --contains HEAD | grep '^\* ' | sed -e 's/^\* //')
+
+if grep -q "^Autobuild\-User($branch): " "$1"; then
+	echo "Already marked as tested for $branch"
+	exit 0
+fi
+
+fullname=$(getent passwd $USER | cut -d: -f5 | cut -d',' -f1)
+mailaddr=$(git config user.email)
+if test -z "$mailaddr"; then
+	mailaddr="$USER@samba.org"
+fi
+cat <<EOF >>"$1"
+
+Autobuild-User($branch): $fullname <$mailaddr>
+Autobuild-Date($branch): $(date) on $(hostname)
+EOF
+exit 0
diff --git a/script/compare_cc_results.py b/script/compare_cc_results.py
new file mode 100755
index 0000000..9bf24ad
--- /dev/null
+++ b/script/compare_cc_results.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Compare the results of native and cross-compiled configure tests
+
+The compared files are called "default.cache.py" and are generated in
+bin/c4che/.
+
+USAGE: compare_cc_results.py CONFIG_1 CONFIG_2 [CONFIG_3 [CONFIG_4 ...]]
+"""
+import sys
+import difflib
+
+exceptions = [
+    'BUILD_DIRECTORY', 'SELFTEST_PREFIX', 'defines',
+    'CROSS_COMPILE', 'CROSS_ANSWERS', 'CROSS_EXECUTE',
+    'LIBSOCKET_WRAPPER_SO_PATH',
+    'LIBNSS_WRAPPER_SO_PATH',
+    'LIBPAM_WRAPPER_SO_PATH',
+    'PAM_SET_ITEMS_SO_PATH',
+    'LIBUID_WRAPPER_SO_PATH',
+    'LIBRESOLV_WRAPPER_SO_PATH',
+]
+
+if len(sys.argv) < 3:
+    print(__doc__)
+    sys.exit(1)
+
+base_lines = list()
+base_fname = ''
+
+found_diff = False
+
+for fname in sys.argv[1:]:
+    lines = list()
+    f = open(fname, 'r')
+    for line in f:
+        if line.startswith("cfg_files ="):
+            # waf writes configuration files as absolute paths
+            continue
+        if len(line.split('=', 1)) == 2:
+            key = line.split('=', 1)[0].strip()
+            value = line.split('=', 1)[1].strip()
+            if key in exceptions:
+                continue
+            # using waf with python 3.4 seems to randomly sort dict keys
+            # we can't modify the waf code but we can fake a dict value
+            # string representation as if it were sorted. python 3.6.5
+            # doesn't seem to suffer from this behaviour
+            if value.startswith('{'):
+                import ast
+                amap = ast.literal_eval(value)
+                fakeline = ""
+                for k in sorted(amap.keys()):
+                    if not len(fakeline) == 0:
+                        fakeline = fakeline + ", "
+                    fakeline = fakeline + '\'' + k + '\': \'' + amap[k] + '\''
+                line = key + ' = {' + fakeline + '}'
+        lines.append(line)
+    f.close()
+    if base_fname:
+        diff = list(difflib.unified_diff(base_lines, lines, base_fname, fname))
+        if diff:
+            print('configuration files %s and %s do not match' % (base_fname, fname))
+            for l in diff:
+                sys.stdout.write(l)
+            found_diff = True
+    else:
+        base_fname = fname
+        base_lines = lines
+
+if found_diff:
+    sys.exit(1)
diff --git a/script/configure_check_unused.pl b/script/configure_check_unused.pl
new file mode 100755
index 0000000..52d8dee
--- /dev/null
+++ b/script/configure_check_unused.pl
@@ -0,0 +1,124 @@
+#!/usr/bin/perl
+# Script that finds macros in a configure script that are not 
+# used in a set of C files.
+# Copyright Jelmer Vernooij <jelmer@samba.org>, GPL
+#
+# Usage: ./$ARGV[0] configure.in [c-files...]
+
+use strict;
+
+sub autoconf_parse($$$$)
+{
+	my $in = shift;
+	my $defines = shift;
+	my $functions = shift;
+	my $headers = shift;
+
+	open(IN, $in) or die("Can't open $in");
+
+	my $ln = 0;
+
+	foreach(<IN>) {
+		$ln++;
+
+		if(/AC_DEFINE\(([^,]+),/) { 
+			$defines->{$1} = "$in:$ln";
+		} 
+	
+		if(/AC_CHECK_FUNCS\(\[*(.[^],)]+)/) {
+			foreach(split / /, $1) { 
+				$functions->{$_} = "$in:$ln";
+			}
+		}
+
+		if(/AC_CHECK_FUNC\(([^,)]+)/) {
+			$functions->{$1} = "$in:$ln";
+		}
+
+		if(/AC_CHECK_HEADERS\(\[*([^],)]+)/) {
+			foreach(split / /, $1) { 
+				$headers->{$_} = "$in:$ln";
+			}
+		}
+
+		if(/AC_CHECK_HEADER\(([^,)]+)/) {
+			$headers->{$1} = "$in:$ln";
+		}
+
+		if(/sinclude\(([^,]+)\)/) {
+			autoconf_parse($1, $defines, $functions, $headers);
+		}
+	}
+
+	close IN;
+}
+
+# Return the symbols and headers used by a C file
+sub cfile_parse($$$)
+{
+	my $in = shift;
+	my $symbols = shift;
+	my $headers = shift;
+
+	open(FI, $in) or die("Can't open $in");
+	my $ln = 0;	
+	my $line;
+	while($line = <FI>) { 
+		$ln++;
+		$_ = $line;
+		if (/\#([ \t]*)include ["<]([^">]+)/) { 
+			$headers->{$2} = "$in:$ln";
+		}
+
+		$_ = $line;
+		while(/([A-Za-z0-9_]+)/g) { 
+			$symbols->{$1} = "$in:$ln";
+		}
+	}
+	close FI;
+}
+
+my %ac_defines = ();
+my %ac_func_checks = ();
+my %ac_headers = ();
+my %symbols = ();
+my %headers = ();
+
+if (scalar(@ARGV) <= 1) {
+	print("Usage: configure_find_unused.pl configure.in [CFILE...]\n");
+	exit 0;
+}
+
+autoconf_parse(shift(@ARGV), \%ac_defines, \%ac_func_checks, \%ac_headers);
+cfile_parse($_, \%symbols, \%headers) foreach(@ARGV);
+
+(keys %ac_defines) or warn("No defines found in configure.in file, parse error?");
+
+foreach (keys %ac_defines) {
+	if (not defined($symbols{$_})) {
+		print "$ac_defines{$_}: Autoconf-defined $_ is unused\n";
+	}
+}
+
+(keys %ac_func_checks) or warn("No function checks found in configure.in file, parse error?");
+
+foreach (keys %ac_func_checks) {
+	my $def = "HAVE_".uc($_);
+	if (not defined($symbols{$_})) {
+		print "$ac_func_checks{$_}: Autoconf-checked function `$_' is unused\n";
+	} elsif (not defined($symbols{$def})) {
+		print "$ac_func_checks{$_}: Autoconf-define `$def' for function `$_' is unused\n";
+	}
+}
+
+(keys %ac_headers) or warn("No headers found in configure.in file, parse error?");
+
+foreach (keys %ac_headers) {
+	my $def = "HAVE_".uc($_);
+	$def =~ s/[\/\.]/_/g;
+	if (not defined($headers{$_})) {
+		print "$ac_headers{$_}: Autoconf-checked header `$_' is unused\n";
+	} elsif (not defined($symbols{$def})) {
+		print "$ac_headers{$_}: Autoconf-define `$def' for header `$_' is unused\n"; 
+	}
+}
diff --git a/script/ctdb-import.msg-filter.sh b/script/ctdb-import.msg-filter.sh
new file mode 100755
index 0000000..107d736
--- /dev/null
+++ b/script/ctdb-import.msg-filter.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+#
+
+set -e
+set -u
+
+cat -
+echo ""
+echo "(This used to be ctdb commit ${GIT_COMMIT})"
+
+exit 0
diff --git a/script/ctdb-import.tree-filter.sh b/script/ctdb-import.tree-filter.sh
new file mode 100755
index 0000000..00b6255
--- /dev/null
+++ b/script/ctdb-import.tree-filter.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+
+set -e
+set -u
+
+lo=$(find -mindepth 1 -maxdepth 1)
+for o in $lo; do
+	mkdir -p ctdb
+	mv $o ctdb/
+done
+
+exit 0
diff --git a/script/ctdb-import.txt b/script/ctdb-import.txt
new file mode 100644
index 0000000..621b24e
--- /dev/null
+++ b/script/ctdb-import.txt
@@ -0,0 +1,5 @@
+ctdb-import.git$ git filter-branch \
+		--tree-filter /path/to/ctdb-import.tree-filter.sh \
+		--msg-filter  /path/to/ctdb-import.msg-filter.sh \
+		HEAD
+
diff --git a/script/find_python.sh b/script/find_python.sh
new file mode 100755
index 0000000..5ef8368
--- /dev/null
+++ b/script/find_python.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+if [ $# -lt 1 ]; then
+	echo "$0: <installdir>"
+	exit 1
+fi
+
+installdir=$1
+exit $(find ${installdir} -name \*.py | wc -l)
diff --git a/script/findstatic.pl b/script/findstatic.pl
new file mode 100755
index 0000000..43a4916
--- /dev/null
+++ b/script/findstatic.pl
@@ -0,0 +1,70 @@
+#!/usr/bin/perl -w
+# find a list of fns and variables in the code that could be static
+# usually called with something like this:
+#    findstatic.pl `find . -name "*.o"`
+# Andrew Tridgell <tridge@samba.org>
+
+use strict;
+
+# use nm to find the symbols
+my($saved_delim) = $/;
+undef $/;
+my($syms) = `nm -o @ARGV`;
+$/ = $saved_delim;
+
+my(@lines) = split(/\n/s, $syms);
+
+my(%def);
+my(%undef);
+my(%stype);
+
+my(%typemap) = (
+	       "T" => "function",
+	       "C" => "uninitialised variable",
+	       "D" => "initialised variable"
+		);
+
+
+# parse the symbols into defined and undefined 
+for (my($i)=0; $i <= $#{@lines}; $i++) {
+	my($line) = $lines[$i];
+	if ($line =~ /(.*):[a-f0-9]* ([TCD]) (.*)/) {
+		my($fname) = $1;
+		my($symbol) = $3;
+		push(@{$def{$fname}}, $symbol);
+		$stype{$symbol} = $2;
+	}
+	if ($line =~ /(.*):\s* U (.*)/) {
+		my($fname) = $1;
+		my($symbol) = $2;
+		push(@{$undef{$fname}}, $symbol);
+	}
+}
+
+# look for defined symbols that are never referenced outside the place they 
+# are defined
+foreach my $f (keys %def) {
+	print "Checking $f\n";
+	my($found_one) = 0;
+	foreach my $s (@{$def{$f}}) {
+		my($found) = 0;
+		foreach my $f2 (keys %undef) {
+			if ($f2 ne $f) {
+				foreach my $s2 (@{$undef{$f2}}) {
+					if ($s2 eq $s) {
+						$found = 1;
+						$found_one = 1;
+					}
+				}
+			}
+		}
+		if ($found == 0) {
+			my($t) = $typemap{$stype{$s}};
+			print "  '$s' is unique to $f  ($t)\n";
+		}
+	}
+	if ($found_one == 0) {
+		print "  all symbols in '$f' are unused (main program?)\n";
+	}
+}
+
diff --git a/script/generate_param.py b/script/generate_param.py
new file mode 100644
index 0000000..50f2d12
--- /dev/null
+++ b/script/generate_param.py
@@ -0,0 +1,431 @@
+# Unix SMB/CIFS implementation.
+# Copyright (C) 2014 Catalyst.Net Ltd
+#
+# Auto generate param_functions.c
+#
+#   ** NOTE! The following LGPL license applies to the ldb
+#   ** library. This does NOT imply that all of Samba is released
+#   ** under the LGPL
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 3 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import xml.etree.ElementTree as ET
+import optparse
+
+# parse command line arguments
+parser = optparse.OptionParser()
+parser.add_option("-f", "--file", dest="filename",
+                  help="input file", metavar="FILE")
+parser.add_option("-o", "--output", dest="output",
+                  help='output file', metavar="FILE")
+parser.add_option("--mode", type="choice", metavar="<FUNCTIONS|S3PROTO|LIBPROTO|PARAMDEFS|PARAMTABLE>",
+                  choices=["FUNCTIONS", "S3PROTO", "LIBPROTO", "PARAMDEFS", "PARAMTABLE"], default="FUNCTIONS")
+parser.add_option("--scope", metavar="<GLOBAL|LOCAL>",
+                  choices=["GLOBAL", "LOCAL"], default="GLOBAL")
+
+(options, args) = parser.parse_args()
+
+if options.filename is None:
+    parser.error("No input file specified")
+if options.output is None:
+    parser.error("No output file specified")
+
+
+def iterate_all(path):
+    """Iterate and yield all the parameters.
+
+    :param path: path to parameters xml file
+    """
+
+    try:
+        with open(path, 'r') as p:
+            out = p.read()
+    except IOError as e:
+        raise Exception("Error opening parameters file")
+
+    # parse the parameters xml file
+    root = ET.fromstring(out)
+    for parameter in root:
+        name = parameter.attrib.get("name")
+        param_type = parameter.attrib.get("type")
+        context = parameter.attrib.get("context")
+        func = parameter.attrib.get("function")
+        synonym = parameter.attrib.get("synonym")
+        removed = parameter.attrib.get("removed")
+        generated = parameter.attrib.get("generated_function")
+        handler = parameter.attrib.get("handler")
+        enumlist = parameter.attrib.get("enumlist")
+        deprecated = parameter.attrib.get("deprecated")
+        synonyms = parameter.findall('synonym')
+
+        if removed == "1":
+            continue
+
+        constant = parameter.attrib.get("constant")
+        substitution = parameter.attrib.get("substitution")
+        parm = parameter.attrib.get("parm")
+        if name is None or param_type is None or context is None:
+            raise Exception("Error parsing parameter: " + name)
+        if func is None:
+            func = name.replace(" ", "_").lower()
+        if enumlist is None:
+            enumlist = "NULL"
+        if handler is None:
+            handler = "NULL"
+        yield {'name': name,
+               'type': param_type,
+               'context': context,
+               'function': func,
+               'constant': (constant == '1'),
+               'substitution': (substitution == '1'),
+               'parm': (parm == '1'),
+               'synonym' : synonym,
+               'generated' : generated,
+               'enumlist' : enumlist,
+               'handler' : handler,
+               'deprecated' : deprecated,
+               'synonyms' : synonyms }
+
+
+# map doc attributes to a section of the generated function
+context_dict = {"G": "_GLOBAL", "S": "_LOCAL"}
+param_type_dict = {
+                    "boolean"      : "_BOOL",
+                    "list"         : "_LIST",
+                    "string"       : "_STRING",
+                    "integer"      : "_INTEGER",
+                    "enum"         : "_INTEGER",
+                    "char"         : "_CHAR",
+                    "boolean-auto" : "_INTEGER",
+                    "cmdlist"      : "_LIST",
+                    "bytes"        : "_INTEGER",
+                    "octal"        : "_INTEGER",
+                    "ustring"      : "_STRING",
+                  }
+
+
+def generate_functions(path_in, path_out):
+    f = open(path_out, 'w')
+    try:
+        f.write('/* This file was automatically generated by generate_param.py. DO NOT EDIT */\n\n')
+        for parameter in iterate_all(options.filename):
+            # filter out parameteric options
+            if ':' in parameter['name']:
+                continue
+            if parameter['synonym'] == "1":
+                continue
+            if parameter['generated'] == "0":
+                continue
+
+            output_string = "FN"
+            temp = context_dict.get(parameter['context'])
+            if temp is None:
+                raise Exception(parameter['name'] + " has an invalid context " + parameter['context'])
+            output_string += temp
+            if parameter['type'] == "string" or parameter['type'] == "ustring":
+                if parameter['substitution']:
+                    output_string += "_SUBSTITUTED"
+                else:
+                    output_string += "_CONST"
+            if parameter['parm']:
+                output_string += "_PARM"
+            temp = param_type_dict.get(parameter['type'])
+            if temp is None:
+                raise Exception(parameter['name'] + " has an invalid param type " + parameter['type'])
+            output_string += temp
+            f.write(output_string + "(" + parameter['function'] + ", " + parameter['function'] + ')\n')
+    finally:
+        f.close()
+
+
+mapping = {
+            'boolean'      : 'bool ',
+            'string'       : 'char *',
+            'integer'      : 'int ',
+            'char'         : 'char ',
+            'list'         : 'const char **',
+            'enum'         : 'int ',
+            'boolean-auto' : 'int ',
+            'cmdlist'      : 'const char **',
+            'bytes'        : 'int ',
+            'octal'        : 'int ',
+            'ustring'      : 'char *',
+          }
+
+
+def make_s3_param_proto(path_in, path_out):
+    file_out = open(path_out, 'w')
+    try:
+        file_out.write('/* This file was automatically generated by generate_param.py. DO NOT EDIT */\n\n')
+        header = get_header(path_out)
+        file_out.write("#ifndef %s\n" % header)
+        file_out.write("#define %s\n\n" % header)
+        file_out.write("struct share_params;\n")
+        file_out.write("struct loadparm_substitution;\n")
+        for parameter in iterate_all(path_in):
+            # filter out parameteric options
+            if ':' in parameter['name']:
+                continue
+            if parameter['synonym'] == "1":
+                continue
+            if parameter['generated'] == "0":
+                continue
+
+            output_string = ""
+            param_type = mapping.get(parameter['type'])
+            if param_type is None:
+                raise Exception(parameter['name'] + " has an invalid context " + parameter['context'])
+            output_string += param_type
+            output_string += "lp_%s" % parameter['function']
+
+            param = None
+            if parameter['parm']:
+                param = "const struct share_params *p"
+            else:
+                param = "int"
+
+            if parameter['type'] == 'string' or parameter['type'] == 'ustring':
+                if parameter['substitution']:
+                    if parameter['context'] == 'G':
+                        output_string += '(TALLOC_CTX *ctx, const struct loadparm_substitution *lp_sub);\n'
+                    elif parameter['context'] == 'S':
+                        output_string += '(TALLOC_CTX *ctx, const struct loadparm_substitution *lp_sub, %s);\n' % param
+                    else:
+                        raise Exception(parameter['name'] + " has an invalid param type " + parameter['type'])
+                else:
+                    if parameter['context'] == 'G':
+                        output_string = 'const ' + output_string + '(void);\n'
+                    elif parameter['context'] == 'S':
+                        output_string = 'const ' + output_string + '(%s);\n' % param
+                    else:
+                        raise Exception(parameter['name'] + " has an invalid param type " + parameter['type'])
+            else:
+                if parameter['context'] == 'G':
+                    output_string += '(void);\n'
+                elif parameter['context'] == 'S':
+                    output_string += '(%s);\n' % param
+                else:
+                    raise Exception(parameter['name'] + " has an invalid param type " + parameter['type'])
+
+            file_out.write(output_string)
+
+        file_out.write("\n#endif /* %s */\n\n" % header)
+    finally:
+        file_out.close()
+
+
+def make_lib_proto(path_in, path_out):
+    file_out = open(path_out, 'w')
+    try:
+        file_out.write('/* This file was automatically generated by generate_param.py. DO NOT EDIT */\n\n')
+        for parameter in iterate_all(path_in):
+            # filter out parameteric options
+            if ':' in parameter['name']:
+                continue
+            if parameter['synonym'] == "1":
+                continue
+            if parameter['generated'] == "0":
+                continue
+
+            output_string = ""
+            param_type = mapping.get(parameter['type'])
+            if param_type is None:
+                raise Exception(parameter['name'] + " has an invalid context " + parameter['context'])
+            output_string += param_type
+
+            output_string += "lpcfg_%s" % parameter['function']
+
+            if parameter['type'] == 'string' or parameter['type'] == 'ustring':
+                if parameter['substitution']:
+                    if parameter['context'] == 'G':
+                        output_string += '(struct loadparm_context *, const struct loadparm_substitution *lp_sub, TALLOC_CTX *ctx);\n'
+                    elif parameter['context'] == 'S':
+                        output_string += '(struct loadparm_service *, struct loadparm_service *, TALLOC_CTX *ctx);\n'
+                    else:
+                        raise Exception(parameter['name'] + " has an invalid context " + parameter['context'])
+                else:
+                    if parameter['context'] == 'G':
+                        output_string = 'const ' + output_string + '(struct loadparm_context *);\n'
+                    elif parameter['context'] == 'S':
+                        output_string = 'const ' + output_string + '(struct loadparm_service *, struct loadparm_service *);\n'
+                    else:
+                        raise Exception(parameter['name'] + " has an invalid param type " + parameter['type'])
+            else:
+                if parameter['context'] == 'G':
+                    output_string += '(struct loadparm_context *);\n'
+                elif parameter['context'] == 'S':
+                    output_string += '(struct loadparm_service *, struct loadparm_service *);\n'
+                else:
+                    raise Exception(parameter['name'] + " has an invalid param type " + parameter['type'])
+
+            file_out.write(output_string)
+    finally:
+        file_out.close()
+
+
+def get_header(path):
+    header = os.path.basename(path).upper()
+    header = header.replace(".", "_").replace("\\", "_").replace("-", "_")
+    return "__%s__" % header
+
+
+def make_param_defs(path_in, path_out, scope):
+    file_out = open(path_out, 'w')
+    try:
+        file_out.write('/* This file was automatically generated by generate_param.py. DO NOT EDIT */\n\n')
+        header = get_header(path_out)
+        file_out.write("#ifndef %s\n" % header)
+        file_out.write("#define %s\n\n" % header)
+        if scope == "GLOBAL":
+            file_out.write("/**\n")
+            file_out.write(" * This structure describes global (ie., server-wide) parameters.\n")
+            file_out.write(" */\n")
+            file_out.write("struct loadparm_global \n")
+            file_out.write("{\n")
+            file_out.write("\tTALLOC_CTX *ctx; /* Context for talloced members */\n")
+        elif scope == "LOCAL":
+            file_out.write("/**\n")
+            file_out.write(" * This structure describes a single service.\n")
+            file_out.write(" */\n")
+            file_out.write("struct loadparm_service \n")
+            file_out.write("{\n")
+            file_out.write("\tbool   autoloaded;\n")
+
+        for parameter in iterate_all(path_in):
+            # filter out parameteric options
+            if ':' in parameter['name']:
+                continue
+            if parameter['synonym'] == "1":
+                continue
+
+            if (scope == "GLOBAL" and parameter['context'] != "G" or
+                scope == "LOCAL" and parameter['context'] != "S"):
+                continue
+
+            output_string = "\t"
+            param_type = mapping.get(parameter['type'])
+            if param_type is None:
+                raise Exception(parameter['name'] + " has an invalid context " + parameter['context'])
+            output_string += param_type
+
+            output_string += "  %s;\n" % parameter['function']
+            file_out.write(output_string)
+
+        file_out.write("LOADPARM_EXTRA_%sS\n" % scope)
+        file_out.write("};\n")
+        file_out.write("\n#endif /* %s */\n\n" % header)
+    finally:
+        file_out.close()
+
+
+type_dict = {
+              "boolean"      : "P_BOOL",
+              "boolean-rev"  : "P_BOOLREV",
+              "boolean-auto" : "P_ENUM",
+              "list"         : "P_LIST",
+              "string"       : "P_STRING",
+              "integer"      : "P_INTEGER",
+              "enum"         : "P_ENUM",
+              "char"         : "P_CHAR",
+              "cmdlist"      : "P_CMDLIST",
+              "bytes"        : "P_BYTES",
+              "octal"        : "P_OCTAL",
+              "ustring"      : "P_USTRING",
+            }
+
+
+def make_param_table(path_in, path_out):
+    file_out = open(path_out, 'w')
+    try:
+        file_out.write('/* This file was automatically generated by generate_param.py. DO NOT EDIT */\n\n')
+        header = get_header(path_out)
+        file_out.write("#ifndef %s\n" % header)
+        file_out.write("#define %s\n\n" % header)
+
+        file_out.write("struct parm_struct parm_table[] = {\n")
+
+        for parameter in iterate_all(path_in):
+            # filter out parameteric options
+            if ':' in parameter['name']:
+                continue
+            if parameter['context'] == 'G':
+                p_class = "P_GLOBAL"
+            else:
+                p_class = "P_LOCAL"
+
+            p_type = type_dict.get(parameter['type'])
+
+            if parameter['context'] == 'G':
+                temp = "GLOBAL"
+            else:
+                temp = "LOCAL"
+            offset = "%s_VAR(%s)" % (temp, parameter['function'])
+
+            enumlist = parameter['enumlist']
+            handler = parameter['handler']
+            synonym = parameter['synonym']
+            deprecated = parameter['deprecated']
+            flags_list = []
+            if synonym == "1":
+                flags_list.append("FLAG_SYNONYM")
+            if deprecated == "1":
+                flags_list.append("FLAG_DEPRECATED")
+            flags = "|".join(flags_list)
+            synonyms = parameter['synonyms']
+
+            file_out.write("\t{\n")
+            file_out.write("\t\t.label\t\t= \"%s\",\n" % parameter['name'])
+            file_out.write("\t\t.type\t\t= %s,\n" % p_type)
+            file_out.write("\t\t.p_class\t= %s,\n" % p_class)
+            file_out.write("\t\t.offset\t\t= %s,\n" % offset)
+            file_out.write("\t\t.special\t= %s,\n" % handler)
+            file_out.write("\t\t.enum_list\t= %s,\n" % enumlist)
+            if flags != "":
+                file_out.write("\t\t.flags\t\t= %s,\n" % flags)
+            file_out.write("\t},\n")
+
+            if synonyms is not None:
+                # for synonyms, we only list the synonym flag:
+                flags = "FLAG_SYNONYM"
+                for syn in synonyms:
+                    file_out.write("\t{\n")
+                    file_out.write("\t\t.label\t\t= \"%s\",\n" % syn.text)
+                    file_out.write("\t\t.type\t\t= %s,\n" % p_type)
+                    file_out.write("\t\t.p_class\t= %s,\n" % p_class)
+                    file_out.write("\t\t.offset\t\t= %s,\n" % offset)
+                    file_out.write("\t\t.special\t= %s,\n" % handler)
+                    file_out.write("\t\t.enum_list\t= %s,\n" % enumlist)
+                    if flags != "":
+                        file_out.write("\t\t.flags\t\t= %s,\n" % flags)
+                    file_out.write("\t},\n")
+
+        file_out.write("\n\t{ .label = NULL }\n")
+        file_out.write("};\n")
+        file_out.write("\n#endif /* %s */\n\n" % header)
+    finally:
+        file_out.close()
+
+
+if options.mode == 'FUNCTIONS':
+    generate_functions(options.filename, options.output)
+elif options.mode == 'S3PROTO':
+    make_s3_param_proto(options.filename, options.output)
+elif options.mode == 'LIBPROTO':
+    make_lib_proto(options.filename, options.output)
+elif options.mode == 'PARAMDEFS':
+    make_param_defs(options.filename, options.output, options.scope)
+elif options.mode == 'PARAMTABLE':
+    make_param_table(options.filename, options.output)
diff --git a/script/git-hooks/check-trailing-whitespace b/script/git-hooks/check-trailing-whitespace
new file mode 100755
index 0000000..4dc1a6d
--- /dev/null
+++ b/script/git-hooks/check-trailing-whitespace
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+git diff-index --cached --check HEAD -- :/*.[ch] :/*.p[ylm]
+
+if [ $? != 0 ]; then
+	echo
+	echo "The commit failed because it seems to introduce trailing whitespace"
+	echo "into C, Perl, or Python code."
+	echo
+	echo "If you are sure you want to do this, repeat the commit with the "
+	echo "--no-verify, like this:"
+	echo
+	echo "   git commit --no-verify"
+	exit 1
+fi
+
+exit 0
diff --git a/script/git-hooks/pre-commit-hook b/script/git-hooks/pre-commit-hook
new file mode 100755
index 0000000..1bcb000
--- /dev/null
+++ b/script/git-hooks/pre-commit-hook
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set -eu
+
+gitdir=$(git rev-parse --show-toplevel)
+if [ $? -ne 0 ]; then
+	echo "git rev-parse --show-toplevel failed"
+	exit 1
+fi
+
+if [ ! -f ${gitdir}/script/git-hooks/pre-commit-script ]; then
+	exit 0
+fi
+
+${gitdir}/script/git-hooks/pre-commit-script || exit $?
+
+exit 0
diff --git a/script/git-hooks/pre-commit-script b/script/git-hooks/pre-commit-script
new file mode 100755
index 0000000..8adb01c
--- /dev/null
+++ b/script/git-hooks/pre-commit-script
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -eu
+
+#
+# make emacs/magit work, cf
+# https://github.com/magit/magit/issues/3419
+#
+unset GIT_LITERAL_PATHSPECS
+
+gitdir=$(git rev-parse --show-toplevel)
+if [ $? -ne 0 ]; then
+	echo "git rev-parse --show-toplevel failed"
+	exit 1
+fi
+
+${gitdir}/script/git-hooks/check-trailing-whitespace || exit $?
+
+exit 0
diff --git a/script/identity_cc.sh b/script/identity_cc.sh
new file mode 100755
index 0000000..8b5118e
--- /dev/null
+++ b/script/identity_cc.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+#An "identity cross-execute" script
+#It can be used for testing the cross-build infrastructure
+#as follows:
+#./configure --cross-compile --cross-execute=./script/identity_cc.sh
+#If the build is actually a native build, then the configuration
+#result should be just like running ./configure without --cross-compile.
+
+eval "$@"
diff --git a/script/release.sh b/script/release.sh
new file mode 100755
index 0000000..b144f74
--- /dev/null
+++ b/script/release.sh
@@ -0,0 +1,1275 @@
+#!/bin/bash
+# make a release of Samba or a library
+
+LC_ALL=C
+export LC_ALL
+LANG=C
+export LANG
+LANGUAGE=C
+export LANGUAGE
+
+set -u
+set -e
+umask 0022
+
+CONF_REPO_URL="ssh://git.samba.org/data/git/samba.git"
+CONF_UPLOAD_URL="samba-bugs@download-master.samba.org:/home/data/ftp/pub"
+CONF_DOWNLOAD_URL="https://download.samba.org/pub"
+CONF_HISTORY_URL="https://www.samba.org"
+
+test -d ".git" -o -r ".git" || {
+	echo "Run this script from the top-level directory in the"
+	echo "repository"
+	exit 1
+}
+
+usage()
+{
+	echo "Usage: script/release.sh <PRODUCT> <COMMAND>"
+	echo ""
+	echo "PRODUCT: ldb, talloc, tevent, tdb, samba-rc, samba-stable"
+	echo "COMMAND: fullrelease, create, push, upload, announce"
+	echo ""
+	return 0
+}
+
+test -x "script/release.sh" || {
+	usage
+	echo "Run this script from the top-level directory in the"
+	echo "repository: as 'script/release.sh'"
+	exit 1
+}
+
+check_args()
+{
+	local cmd="$1"
+	local got_args="$2"
+	local take_args="$3"
+
+	test x"${got_args}" = x"${take_args}" || {
+		usage
+		echo "cmd[${cmd}] takes ${take_args} instead of ${got_args}"
+		return 1
+	}
+
+	return 0
+}
+
+min_args()
+{
+	local cmd="$1"
+	local got_args="$2"
+	local min_args="$3"
+
+	test "${got_args}" -ge "${min_args}" || {
+		usage
+		echo "cmd[${cmd}] takes at least ${min_args} instead of ${got_args}"
+		return 1
+	}
+
+	return 0
+}
+
+min_args "$0" "$#" "2"
+
+product="$1"
+globalcmd="$2"
+shift 2
+oldtagname=""
+tagname=""
+patchfile=""
+cmds=""
+next_cmd=""
+
+require_tagname()
+{
+	min_args "${FUNCNAME}" "$#" "1" || return 1
+	local cmd="$1"
+
+	test -n "${tagname}" || {
+		echo "cmd[${cmd}] requires '\${tagname}' variable to be set"
+		return 1
+	}
+
+	local name=$(echo "${tagname}" | cut -d '-' -f1)
+	test x"${name}" = x"${productbase}" || {
+		echo "Invalid tagname[${tgzname}]"
+		return 1
+	}
+
+	return 0
+}
+
+cmd_allowed()
+{
+	min_args "${FUNCNAME}" "$#" "2" || return 1
+	local cmd="$1"
+	shift 1
+
+	echo "$@" | grep -q "\<${cmd}\>" || {
+		return 1
+	}
+
+	return 0
+}
+
+verify_samba_rc()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+
+	test -f VERSION || {
+		echo "VERSION doesn't exist"
+		return 1
+	}
+
+	grep -q 'SAMBA_VERSION_IS_GIT_SNAPSHOT=no' VERSION || {
+		echo "SAMBA_VERSION_IS_GIT_SNAPSHOT is not 'no'"
+		return 1
+	}
+
+	grep -q '^SAMBA_VERSION_RC_RELEASE=' VERSION || {
+		echo "SAMBA_VERSION_RC_RELEASE= missing"
+		return 1
+	}
+
+	grep -q '^SAMBA_VERSION_RC_RELEASE=$' VERSION && {
+		echo "SAMBA_VERSION_RC_RELEASE= missing the rc version"
+		return 1
+	}
+
+	return 0
+}
+
+load_samba_stable_versions()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+
+	test -n "${version-}" && {
+		return 0
+	}
+
+	local SAMBA_VERSION_MAJOR=$(grep '^SAMBA_VERSION_MAJOR=' VERSION | cut -d '=' -f2 | xargs)
+	local SAMBA_VERSION_MINOR=$(grep '^SAMBA_VERSION_MINOR=' VERSION | cut -d '=' -f2 | xargs)
+	local SAMBA_VERSION_RELEASE=$(grep '^SAMBA_VERSION_RELEASE=' VERSION | cut -d '=' -f2 | xargs)
+
+	version="${SAMBA_VERSION_MAJOR}.${SAMBA_VERSION_MINOR}.${SAMBA_VERSION_RELEASE}"
+	tagname="${productbase}-${version}"
+
+	test ${SAMBA_VERSION_RELEASE} -gt 0 || {
+		return 0
+	}
+
+	oldversion="${SAMBA_VERSION_MAJOR}.${SAMBA_VERSION_MINOR}.$(expr ${SAMBA_VERSION_RELEASE} - 1)"
+	oldtagname="${productbase}-${oldversion}"
+	patchfile="${productbase}-${oldversion}-${version}.diffs"
+
+	return 0
+}
+
+verify_samba_stable()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+
+	test -f VERSION || {
+		echo "VERSION doesn't exist"
+		return 1
+	}
+
+	grep -q 'SAMBA_VERSION_IS_GIT_SNAPSHOT=no' VERSION || {
+		echo "SAMBA_VERSION_IS_GIT_SNAPSHOT is not 'no'"
+		return 1
+	}
+
+	local VARS=""
+	VARS="${VARS} SAMBA_VERSION_REVISION"
+	VARS="${VARS} SAMBA_VERSION_TP_RELEASE"
+	VARS="${VARS} SAMBA_VERSION_ALPHA_RELEASE"
+	VARS="${VARS} SAMBA_VERSION_BETA_RELEASE"
+	VARS="${VARS} SAMBA_VERSION_PRE_RELEASE"
+	VARS="${VARS} SAMBA_VERSION_RC_RELEASE"
+	VARS="${VARS} SAMBA_VERSION_RELEASE_NICKNAME"
+	VARS="${VARS} SAMBA_VERSION_VENDOR_SUFFIX"
+	VARS="${VARS} SAMBA_VERSION_VENDOR_PATCH"
+	for var in ${VARS}; do
+		grep -q "^${var}" VERSION && {
+			grep -q "^${var}=$" VERSION || {
+				echo "${var} found in stable version"
+				return 1
+			}
+		}
+	done
+
+	load_samba_stable_versions
+
+	test -n "${oldtagname}" || {
+		return 0
+	}
+
+	local verify_out="${TMPDIR}/verify-${oldtagname}.out"
+
+	echo "Verifying oldtagname: ${oldtagname}"
+
+	git tag -v "${oldtagname}" >${verify_out} 2>&1 || {
+		echo "failed to verify old tag[${oldtagname}]"
+		echo ""
+		cat "${verify_out}"
+		return 1
+	}
+
+	grep -q "${GPG_KEYID}" "${verify_out}" || {
+		echo "oldtagname[${oldtagname}] was not generated with GPG_KEYID[${GPG_KEYID}]!"
+		echo ""
+		cat "${verify_out}"
+		return 1
+	}
+
+	echo "Verifying ${oldtagname}.tar.gz and ${oldtagname}.tar.asc"
+
+	test -f "${oldtagname}.tar.gz" || {
+		echo "${oldtagname}.tar.gz does not exist"
+		return 1
+	}
+
+	test -f "${oldtagname}.tar.asc" || {
+		echo "${oldtagname}.tar.asc does not exist"
+		return 1
+	}
+
+	zcat "${oldtagname}.tar.gz" | gpg --verify "${oldtagname}.tar.asc" - 2>${verify_out} || {
+		echo "Failed to verify ${oldtagname}.tar.asc"
+		return 1
+	}
+
+	grep -q "${GPG_KEYID}" "${verify_out}" || {
+		echo "${oldtagname}.tar.asc was not generated with GPG_KEYID[${GPG_KEYID}]!"
+		echo ""
+		cat "${verify_out}"
+		return 1
+	}
+
+	return 0
+}
+
+verify_release()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+
+	test -n "${verify_fn}" || {
+		echo "verify_fn variable empty"
+		return 1
+	}
+
+	echo "Running ${verify_fn}"
+	${verify_fn}
+}
+
+create_release()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+
+	echo "Releasing product ${product}"
+
+	test -n "${tagname}" && {
+		git tag -l "${tagname}" | grep -q "${tagname}" && {
+			echo "tagname[${tagname}] already exist"
+			return 1
+		}
+
+		local _tgzname="${tagname}.tar.gz"
+		test -e "${_tgzname}" && {
+			echo "_tgzname[${_tgzname}] already exist"
+			return 1
+		}
+	}
+
+	echo "Building release tarball"
+	local tgzname=$(make dist 2>&1 | grep ^Created | cut -d' ' -f2)
+	test -f "${tgzname}" || {
+		echo "Failed to create tarball"
+		return 1
+	}
+	CLEANUP_FILES="${CLEANUP_FILES} ${tgzname}"
+
+	local name=$(echo "${tgzname}" | cut -d '-' -f1)
+	test x"${name}" = x"${productbase}" || {
+		echo "Invalid tgzname[${tgzname}]"
+		return 1
+	}
+
+	local _tagname=$(basename ${tgzname} .tar.gz)
+	test -n "${tagname}" && {
+		test x"${_tagname}" = x"${tagname}" || {
+			echo "Invalid tgzname[${tgzname}]"
+			return 1
+		}
+	}
+	tagname="${_tagname}"
+
+	local tarname=$(basename ${tgzname} .gz)
+	echo "Tarball: ${tarname}"
+	gunzip -f ${tgzname} || {
+		echo "Failed to decompress tarball ${tarname}"
+		return 1
+	}
+	test -f "${tarname}" || {
+		echo "Failed to decompress tarball ${tarname}"
+		return 1
+	}
+	CLEANUP_FILES="${CLEANUP_FILES} ${tarname}"
+
+	# tagname is global
+	echo "Tagging as ${tagname}"
+	git tag -u ${GPG_KEYID} -s "${tagname}" -m "${productbase}: tag release ${tagname}" || {
+		return 1
+	}
+	CLEANUP_TAGS="${CLEANUP_TAGS} ${tagname}"
+
+	echo "Signing ${tarname} => ${tarname}.asc"
+	rm -f "${tarname}.asc"
+	gpg --default-key "${GPG_KEYID}" --detach-sign --armor ${tarname} || {
+		return 1
+	}
+	test -f "${tarname}.asc" || {
+		echo "Failed to create signature ${tarname}.asc"
+		return 1
+	}
+	CLEANUP_FILES="${CLEANUP_FILES} ${tarname}.asc"
+	echo "Compressing ${tarname} => ${tgzname}"
+	gzip -f -9 ${tarname}
+	test -f "${tgzname}" || {
+		echo "Failed to compress ${tgzname}"
+		return 1
+	}
+
+	return 0
+}
+
+patch_release()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+	require_tagname "${FUNCNAME}"
+
+	test -n "${patchfile}" || {
+		return 0
+	}
+
+	local oldpwd=$(pwd)
+	echo "Generating ${patchfile}"
+	(
+		set -e
+		set -u
+		pushd "${TMPDIR}"
+		tar xfz "${oldpwd}/${oldtagname}.tar.gz"
+		tar xfz "${oldpwd}/${tagname}.tar.gz"
+		diff -Npur "${oldtagname}/" "${tagname}/" >"${patchfile}"
+		popd
+	)
+	CLEANUP_FILES="${CLEANUP_FILES} ${patchfile}"
+	mv "${TMPDIR}/${patchfile}" "${patchfile}" || {
+		echo "failed cmd[mv ${TMPDIR}/${patchfile} ${patchfile}]"
+		return 1
+	}
+
+	echo "Signing ${patchfile} => ${patchfile}.asc"
+	rm -f "${patchfile}.asc"
+	CLEANUP_FILES="${CLEANUP_FILES} ${patchfile}.asc"
+	gpg --default-key "${GPG_KEYID}" --detach-sign --armor ${patchfile} || {
+		return 1
+	}
+	test -f "${patchfile}.asc" || {
+		echo "Failed to create signature ${patchfile}.asc"
+		return 1
+	}
+	echo "Compressing ${patchfile} => ${patchfile}.gz"
+	CLEANUP_FILES="${CLEANUP_FILES} ${patchfile}.gz"
+	gzip -f -9 ${patchfile}
+	test -f "${patchfile}.gz" || {
+		echo "Failed to compress ${patchfile}.gz"
+		return 1
+	}
+
+	return 0
+}
+
+whatsnew_release()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+	require_tagname "${FUNCNAME}"
+
+	echo "extract ${tagname}.WHATSNEW.txt"
+	tar xf ${tagname}.tar.gz --to-stdout ${tagname}/WHATSNEW.txt >${tagname}.WHATSNEW.txt
+	CLEANUP_FILES="${CLEANUP_FILES} ${tagname}.WHATSNEW.txt"
+
+	return 0
+}
+
+check_nopatch()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+	require_tagname "${FUNCNAME}"
+
+	local verify_out="${TMPDIR}/verify-${oldtagname}.out"
+
+	echo "Verifying tagname: ${tagname}"
+
+	git tag -v "${tagname}" >${verify_out} 2>&1 || {
+		echo "failed to verify tag[${tagname}]"
+		echo ""
+		cat "${verify_out}"
+		return 1
+	}
+	grep -q "${GPG_KEYID}" "${verify_out}" || {
+		echo "tagname[${tagname}] was not generated with GPG_KEYID[${GPG_KEYID}]!"
+		echo ""
+		cat "${verify_out}"
+		return 1
+	}
+
+	echo "Verifying ${tagname}.tar.gz and ${tagname}.tar.asc"
+
+	test -f "${tagname}.tar.gz" || {
+		echo "${tagname}.tar.gz does not exist"
+		return 1
+	}
+
+	test -f "${tagname}.tar.asc" || {
+		echo "${tagname}.tar.asc does not exist"
+		return 1
+	}
+
+	zcat "${tagname}.tar.gz" | gpg --verify "${tagname}.tar.asc" - 2>${verify_out} || {
+		echo "Failed to verify ${tagname}.tar.asc"
+		return 1
+	}
+	grep -q "${GPG_KEYID}" "${verify_out}" || {
+		echo "${tagname}.tar.asc was not generated with GPG_KEYID[${GPG_KEYID}]!"
+		echo ""
+		cat "${verify_out}"
+		return 1
+	}
+
+	ls -la ${tagname}.*
+
+	return 0
+}
+
+check_samba_stable()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+	require_tagname "${FUNCNAME}"
+
+	load_samba_stable_versions
+
+	local verify_out="${TMPDIR}/verify-${oldtagname}.out"
+
+	echo "Verifying tagname: ${tagname}"
+
+	git tag -v "${tagname}" >${verify_out} 2>&1 || {
+		echo "failed to verify tag[${tagname}]"
+		echo ""
+		cat "${verify_out}"
+		return 1
+	}
+	grep -q "${GPG_KEYID}" "${verify_out}" || {
+		echo "tagname[${tagname}] was not generated with GPG_KEYID[${GPG_KEYID}]!"
+		echo ""
+		cat "${verify_out}"
+		return 1
+	}
+
+	echo "Verifying ${tagname}.tar.gz and ${tagname}.tar.asc"
+
+	test -f "${tagname}.tar.gz" || {
+		echo "${tagname}.tar.gz does not exist"
+		return 1
+	}
+
+	test -f "${tagname}.tar.asc" || {
+		echo "${tagname}.tar.asc does not exist"
+		return 1
+	}
+
+	zcat "${tagname}.tar.gz" | gpg --verify "${tagname}.tar.asc" - 2>${verify_out} || {
+		echo "Failed to verify ${tagname}.tar.asc"
+		return 1
+	}
+	grep -q "${GPG_KEYID}" "${verify_out}" || {
+		echo "${tagname}.tar.asc was not generated with GPG_KEYID[${GPG_KEYID}]!"
+		echo ""
+		cat "${verify_out}"
+		return 1
+	}
+
+	test -n "${patchfile}" || {
+		ls -lart ${tagname}.*
+		return 0
+	}
+
+	echo "Verifying ${patchfile}.gz and ${patchfile}.asc"
+
+	test -f "${patchfile}.gz" || {
+		echo "${patchfile}.gz does not exist"
+		return 1
+	}
+
+	test -f "${patchfile}.asc" || {
+		echo "${patchfile}.asc does not exist"
+		return 1
+	}
+
+	zcat "${patchfile}.gz" | gpg --verify "${patchfile}.asc" - 2>${verify_out} || {
+		echo "Failed to verify ${patchfile}.asc"
+		return 1
+	}
+	grep -q "${GPG_KEYID}" "${verify_out}" || {
+		echo "${patchfile}.asc was not generated with GPG_KEYID[${GPG_KEYID}]!"
+		echo ""
+		cat "${verify_out}"
+		return 1
+	}
+
+	ls -lart ${tagname}.* ${patchfile}.*
+	return 0
+}
+
+check_release()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+
+	test -n "${check_fn}" || {
+		echo "check_fn variable empty"
+		return 1
+	}
+
+	echo "Running ${check_fn}"
+	${check_fn}
+}
+
+push_release()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+	require_tagname "${FUNCNAME}"
+
+	echo "Push git tag ${tagname} to '${repo_url}'"
+	git push "${repo_url}" "refs/tags/${tagname}:refs/tags/${tagname}" || {
+		return 1
+	}
+
+	return 0
+}
+
+upload_nopatch()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+	require_tagname "${FUNCNAME}"
+
+	echo "Upload ${tagname}.* to '${upload_url}'"
+	rsync -Pav --delay-updates ${tagname}.* "${upload_url}/" || {
+		return 1
+	}
+	rsync ${upload_url}/${tagname}.*
+
+	return 0
+}
+
+upload_samba_stable()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+	require_tagname "${FUNCNAME}"
+
+	load_samba_stable_versions
+
+	local release_url="${upload_url}samba/stable/"
+	local patch_url="${upload_url}samba/patches/"
+
+	echo "Upload ${tagname}.tar.* to '${release_url}'"
+	ls -lart ${tagname}.tar.*
+	rsync -Pav --delay-updates ${tagname}.tar.* "${release_url}/" || {
+		return 1
+	}
+	rsync ${release_url}/${tagname}.tar.*
+
+	test -n "${patchfile}" || {
+		return 0
+	}
+
+	echo "Upload ${patchfile}.* to '${patch_url}'"
+	ls -lart ${patchfile}.*
+	rsync -Pav --delay-updates ${patchfile}.* "${patch_url}/" || {
+		return 1
+	}
+	rsync ${patch_url}/${patchfile}.*
+
+	return 0
+}
+
+upload_release()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+
+	test -n "${upload_fn}" || {
+		echo "upload_fn variable empty"
+		return 1
+	}
+
+	echo "Running ${upload_fn}"
+	${upload_fn}
+}
+
+announcement_samba_rc()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+	require_tagname "${FUNCNAME}"
+
+	test -f "${tagname}.WHATSNEW.txt" || {
+		echo "${tagname}.WHATSNEW.txt does not exist"
+		return 1
+	}
+
+	local t=""
+	local version=$(echo "${tagname}" | sed -e 's!^samba-!!')
+	local href="#${version}"
+	local series=$(echo "${version}" | cut -d '.' -f1-2)
+	local rc=$(echo "${version}" | sed -e 's!.*rc\([0-9][0-9]*\)!\1!')
+	local rcname="${rc}th"
+	case "${rc}" in
+	1)
+		rcname="first"
+		;;
+	2)
+		rcname="second"
+		;;
+	3)
+		rcname="third"
+		;;
+	4)
+		rcname="fourth"
+		;;
+	5)
+		rcname="fifth"
+		;;
+	esac
+
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.to.txt"
+	{
+		echo "samba-announce@lists.samba.org, samba@lists.samba.org, samba-technical@lists.samba.org"
+	} >announce.${tagname}.to.txt
+
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.subject.txt"
+	{
+		echo "[Announce] Samba ${version} Available for Download"
+	} >announce.${tagname}.subject.txt
+
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.mail.txt"
+	{
+		cat ${tagname}.WHATSNEW.txt
+		echo ""
+		echo "================"
+		echo "Download Details"
+		echo "================"
+		echo ""
+		echo "The uncompressed tarballs and patch files have been signed"
+		echo "using GnuPG (ID ${GPG_KEYID}).  The source code can be downloaded"
+		echo "from:"
+		echo ""
+		echo "        ${download_url}"
+		echo ""
+		echo "The release notes are available online at:"
+		echo ""
+		echo "        ${download_url}${tagname}.WHATSNEW.txt"
+		echo ""
+		echo "Our Code, Our Bugs, Our Responsibility."
+		echo "(https://bugzilla.samba.org/)"
+		echo ""
+		echo "                        --Enjoy"
+		echo "                        The Samba Team"
+	} >announce.${tagname}.mail.txt
+
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.mutt-arguments.txt"
+	{
+		echo -n "-i announce.${tagname}.mail.txt "
+		echo -n "-s \"$(cat announce.${tagname}.subject.txt | xargs)\" "
+		echo -n "$(cat announce.${tagname}.to.txt | xargs)"
+	} >announce.${tagname}.mutt-arguments.txt
+
+	local headlinefile="posted_news/@UTCTIME@.${version}.headline.html"
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.headline.html"
+	{
+		echo "<!-- BEGIN: ${headlinefile} -->"
+		echo "<li> @UTCDATE@ <a href=\"${href}\">Samba ${version} Available for Download</a></li>"
+		echo "<!-- END: ${headlinefile} -->"
+	} >announce.${tagname}.headline.html
+
+	local bodyfile="posted_news/@UTCTIME@.${version}.body.html"
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.body.html"
+	{
+		echo "<!-- BEGIN: ${bodyfile} -->"
+		echo "<h5><a name=\"${version}\">@UTCDATE@</a></h5>"
+		echo "<p class="headline">Samba ${version} Available for Download</p>"
+		echo "<p>"
+		echo "This is the ${rcname} release candidate of the upcoming Samba ${series} release series."
+		echo "</p>"
+		echo "<p>"
+		echo "The uncompressed tarball has been signed using GnuPG (ID ${GPG_KEYID})."
+		echo "The source code can be <a href=\"${download_url}${tagname}.tar.gz\">downloaded now</a>."
+		echo "See <a href=\"${download_url}${tagname}.WHATSNEW.txt\">the release notes for more info</a>."
+		echo "</p>"
+		echo "<!-- END: ${bodyfile} -->"
+	} >announce.${tagname}.body.html
+
+	local webrepo="${TMPDIR}/webrepo"
+
+	mkdir "${webrepo}" || {
+		return 1
+	}
+	git -C "${webrepo}" init || {
+		return 1
+	}
+
+	mkdir -p "$(dirname ${webrepo}/${headlinefile})" || {
+		return 1
+	}
+	cp -a "announce.${tagname}.headline.html" "${webrepo}/${headlinefile}" || {
+		return 1
+	}
+
+	mkdir -p "$(dirname ${webrepo}/${bodyfile})" || {
+		return 1
+	}
+	cp -a "announce.${tagname}.body.html" "${webrepo}/${bodyfile}" || {
+		return 1
+	}
+
+	git -C "${webrepo}" add "${headlinefile}" "${bodyfile}" || {
+		return 1
+	}
+	git -C "${webrepo}" commit --signoff --message "NEWS[${version}]: Samba ${version} Available for Download" || {
+		return 1
+	}
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.patch.txt"
+	git -C "${webrepo}" format-patch --stdout -1 HEAD >announce.${tagname}.patch.txt || {
+		return 1
+	}
+
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.todo.txt"
+	{
+		ls -lart announce.${tagname}.*
+		echo ""
+		echo "NOTICE:"
+		echo "You need to do the following manual steps in order"
+		echo "to finish the announcement of ${tagname}!"
+		echo ""
+		echo "Change to a samba-web checkout and run"
+		echo "  ./announce_samba_release.sh ${version} $(pwd)/announce.${tagname}.patch.txt"
+		echo ""
+		echo "Once the resulting commit is pushed a cron job will update "
+		echo "the content exported by the webserver every 5-10 mins."
+		echo "Check https://www.samba.org"
+		echo ""
+		echo "If the web content is updated, you need to send the announce mail (gpg signed)."
+		echo "- announce.${tagname}.to.txt contains the mail's recipients for the To: header."
+		echo "- announce.${tagname}.subject.txt contains the mail's subject line."
+		echo "- announce.${tagname}.mail.txt contains the content of the mail body."
+		echo "In case your're using mutt, you can use the following shortcut:"
+		echo "  eval mutt \$(cat announce.${tagname}.mutt-arguments.txt)"
+		echo ""
+		echo "NOTICE: you're not done yet! Read the above instructions carefully!"
+		echo "See: announce.${tagname}.todo.txt"
+		echo ""
+	} >announce.${tagname}.todo.txt
+
+	ls -lart announce.${tagname}.*
+	return 0
+}
+
+announcement_samba_stable()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+	require_tagname "${FUNCNAME}"
+
+	load_samba_stable_versions
+
+	test -f "${tagname}.tar.gz" || {
+		echo "${tagname}.tar.gz does not exist"
+		return 1
+	}
+
+	local release_url="${download_url}samba/stable/"
+	local patch_url="${download_url}samba/patches/"
+
+	echo "extract WHATSNEW.txt"
+	tar xf ${tagname}.tar.gz --to-stdout ${tagname}/WHATSNEW.txt >${TMPDIR}/WHATSNEW.txt
+
+	local t=""
+	local oldversion=$(echo "${oldtagname}" | sed -e 's!^samba-!!')
+	local version=$(echo "${tagname}" | sed -e 's!^samba-!!')
+	local href="#${version}"
+	local series=$(echo "${version}" | cut -d '.' -f1-2)
+	local release=$(echo "${version}" | cut -d '.' -f3)
+	local releasename="latest"
+	case "${release}" in
+	1)
+		releasename="first"
+		;;
+	*)
+		releasename="latest"
+		;;
+	esac
+
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.to.txt"
+	{
+		echo "samba-announce@lists.samba.org, samba@lists.samba.org, samba-technical@lists.samba.org"
+	} >announce.${tagname}.to.txt
+
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.subject.txt"
+	{
+		echo "[Announce] Samba ${version} Available for Download"
+	} >announce.${tagname}.subject.txt
+
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.mail.txt"
+	{
+		local top=$(cat ${TMPDIR}/WHATSNEW.txt | grep -n '^Release notes for older releases follow:' | head -1 | cut -d ':' -f1)
+		test -n "${top}" || {
+			top=$(cat ${TMPDIR}/WHATSNEW.txt | wc -l)
+		}
+		local skip=$(cat ${TMPDIR}/WHATSNEW.txt | grep -n '^[^ ]' | head -1 | cut -d ':' -f1)
+		local headlimit=$(expr ${top} - 1)
+		local taillimit=$(expr ${headlimit} - \( ${skip} - 1 \))
+
+		echo ""
+		echo ""
+		echo "Release Announcements"
+		echo "---------------------"
+		echo ""
+		head -${headlimit} ${TMPDIR}/WHATSNEW.txt | tail -${taillimit}
+		echo ""
+		echo "================"
+		echo "Download Details"
+		echo "================"
+		echo ""
+		echo "The uncompressed tarballs and patch files have been signed"
+		echo "using GnuPG (ID ${GPG_KEYID}).  The source code can be downloaded"
+		echo "from:"
+		echo ""
+		echo "        ${release_url}"
+		echo ""
+		echo "The release notes are available online at:"
+		echo ""
+		echo "        ${history_url}${tagname}.html"
+		echo ""
+		echo "Our Code, Our Bugs, Our Responsibility."
+		echo "(https://bugzilla.samba.org/)"
+		echo ""
+		echo "                        --Enjoy"
+		echo "                        The Samba Team"
+	} >announce.${tagname}.mail.txt
+
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.mutt-arguments.txt"
+	{
+		echo -n "-i announce.${tagname}.mail.txt "
+		echo -n "-s \"$(cat announce.${tagname}.subject.txt | xargs)\" "
+		echo -n "$(cat announce.${tagname}.to.txt | xargs)"
+	} >announce.${tagname}.mutt-arguments.txt
+
+	local htmlfile="history/${tagname}.html"
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.html"
+	{
+		local tmp=$(cat ${TMPDIR}/WHATSNEW.txt | grep -n '^Reporting bugs & Development Discussion' | head -1 | cut -d ':' -f1)
+		local lines=$(expr ${tmp} - 2)
+
+		echo '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"'
+		echo ' "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">'
+		echo '<html xmlns="http://www.w3.org/1999/xhtml">'
+
+		echo "<head>"
+		echo "<title>Samba ${version} - Release Notes</title>"
+		echo "</head>"
+
+		echo "<body>"
+		echo "<H2>Samba ${version} Available for Download</H2>"
+
+		echo "<p>"
+		echo "<a href=\"${release_url}${tagname}.tar.gz\">Samba ${version} (gzipped)</a><br>"
+		echo "<a href=\"${release_url}${tagname}.tar.asc\">Signature</a>"
+		echo "</p>"
+
+		test -n "${patchfile}" && {
+			echo "<p>"
+			echo "<a href=\"${patch_url}${patchfile}.gz\">Patch (gzipped) against Samba ${oldversion}</a><br>"
+			echo "<a href=\"${patch_url}${patchfile}.asc\">Signature</a>"
+			echo "</p>"
+		}
+
+		echo "<p>"
+		echo "<pre>"
+		head -${lines} ${TMPDIR}/WHATSNEW.txt | sed \
+			-e 's!&!\&amp;!g' | sed \
+			-e 's!<!\&lt;!g' \
+			-e 's!>!\&gt;!g' \
+			-e 's!ä!\&auml;!g' \
+			-e 's!Ä!\&Auml;!g' \
+			-e 's!ö!\&ouml;!g' \
+			-e 's!Ö!\&Ouml;!g' \
+			-e 's!ü!\&uuml;!g' \
+			-e 's!Ü!\&Uuml;!g' \
+			-e 's!ß!\&szlig;!g' \
+			-e 's!"!\&quot;!g' \
+			-e "s!'!\&apos;!g" |
+			cat
+		echo "</pre>"
+		echo "</p>"
+
+		echo "</body>"
+		echo "</html>"
+	} >announce.${tagname}.html
+
+	local headlinefile="posted_news/@UTCTIME@.${version}.headline.html"
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.headline.html"
+	{
+		echo "<!-- BEGIN: ${headlinefile} -->"
+		echo "<li> @UTCDATE@ <a href=\"${href}\">Samba ${version} Available for Download</a></li>"
+		echo "<!-- END: ${headlinefile} -->"
+	} >announce.${tagname}.headline.html
+
+	local bodyfile="posted_news/@UTCTIME@.${version}.body.html"
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.body.html"
+	{
+		echo "<!-- BEGIN: ${bodyfile} -->"
+		echo "<h5><a name=\"${version}\">@UTCDATE@</a></h5>"
+		echo "<p class="headline">Samba ${version} Available for Download</p>"
+		echo "<p>"
+		echo "This is the ${releasename} stable release of the Samba ${series} release series."
+		echo "</p>"
+		echo "<p>"
+		echo "The uncompressed tarball has been signed using GnuPG (ID ${GPG_KEYID})."
+		echo "The source code can be <a href=\"${release_url}${tagname}.tar.gz\">downloaded now</a>."
+		test -n "${patchfile}" && {
+			echo "A <a href=\"${patch_url}${patchfile}.gz\">patch against Samba ${oldversion}</a> is also available."
+		}
+		echo "See <a href=\"${history_url}${tagname}.html\">the release notes for more info</a>."
+		echo "</p>"
+		echo "<!-- END: ${bodyfile} -->"
+	} >announce.${tagname}.body.html
+
+	local webrepo="${TMPDIR}/webrepo"
+
+	mkdir "${webrepo}" || {
+		return 1
+	}
+	git -C "${webrepo}" init || {
+		return 1
+	}
+
+	mkdir -p "$(dirname ${webrepo}/${htmlfile})" || {
+		return 1
+	}
+	cp -a "announce.${tagname}.html" "${webrepo}/${htmlfile}" || {
+		return 1
+	}
+
+	mkdir -p "$(dirname ${webrepo}/${headlinefile})" || {
+		return 1
+	}
+	cp -a "announce.${tagname}.headline.html" "${webrepo}/${headlinefile}" || {
+		return 1
+	}
+
+	mkdir -p "$(dirname ${webrepo}/${bodyfile})" || {
+		return 1
+	}
+	cp -a "announce.${tagname}.body.html" "${webrepo}/${bodyfile}" || {
+		return 1
+	}
+
+	git -C "${webrepo}" add "${htmlfile}" "${headlinefile}" "${bodyfile}" || {
+		return 1
+	}
+	git -C "${webrepo}" commit --signoff --message "NEWS[${version}]: Samba ${version} Available for Download" || {
+		return 1
+	}
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.patch.txt"
+	git -C "${webrepo}" format-patch --stdout -1 HEAD >announce.${tagname}.patch.txt || {
+		return 1
+	}
+
+	CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.todo.txt"
+	{
+		ls -lart announce.${tagname}.*
+		echo ""
+		echo "NOTICE:"
+		echo "You need to do the following manual steps in order"
+		echo "to finish the announcement of ${tagname}!"
+		echo ""
+		echo "Change to a samba-web checkout and run"
+		echo "  ./announce_samba_release.sh ${version} $(pwd)/announce.${tagname}.patch.txt"
+		echo ""
+		echo "Once the resulting commit is pushed a cron job will update "
+		echo "the content exported by the webserver every 5-10 mins."
+		echo "Check https://www.samba.org"
+		echo ""
+		echo "If the web content is updated, you need to send the announce mail (gpg signed)."
+		echo "- announce.${tagname}.to.txt contains the mail's recipients for the To: header."
+		echo "- announce.${tagname}.subject.txt contains the mail's subject line."
+		echo "- announce.${tagname}.mail.txt contains the content of the mail body."
+		echo "In case your're using mutt, you can use the following shortcut:"
+		echo "  eval mutt \$(cat announce.${tagname}.mutt-arguments.txt)"
+		echo ""
+		echo "NOTICE: you're not done yet! Read the above instructions carefully!"
+		echo "See: announce.${tagname}.todo.txt"
+		echo ""
+	} >announce.${tagname}.todo.txt
+
+	ls -lart announce.${tagname}.*
+	return 0
+}
+
+announcement_release()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+
+	test -n "${announcement_fn}" || {
+		echo "announcement_fn variable empty"
+		return 1
+	}
+
+	echo "Running ${announcement_fn}"
+	${announcement_fn}
+}
+
+announce_release()
+{
+	check_args "${FUNCNAME}" "$#" "0" || return 1
+	require_tagname "${FUNCNAME}"
+
+	test -f "announce.${tagname}.todo.txt" || {
+		echo "announce.${tagname}.todo.txt does not exist"
+		return 1
+	}
+
+	cat announce.${tagname}.todo.txt
+	return 0
+}
+
+case "${product}" in
+talloc | tdb | tevent | ldb)
+	test -z "${GPG_USER-}" && {
+		GPG_USER='Samba Library Distribution Key <samba-bugs@samba.org>'
+	}
+
+	test -z "${GPG_KEYID-}" && {
+		GPG_KEYID='4793916113084025'
+	}
+
+	productbase="${product}"
+	srcdir="lib/${product}"
+	repo_url="${CONF_REPO_URL}"
+	upload_url="${CONF_UPLOAD_URL}/${product}/"
+	download_url="${CONF_DOWNLOAD_URL}/${product}/"
+
+	check_fn="check_nopatch"
+	upload_fn="upload_nopatch"
+	fullcmds="create check push upload"
+	;;
+samba-rc)
+	test -z "${GPG_USER-}" && {
+		GPG_USER='Samba Distribution Verification Key <samba-bugs@samba.org>'
+	}
+
+	test -z "${GPG_KEYID-}" && {
+		GPG_KEYID='AA99442FB680B620'
+	}
+
+	productbase="samba"
+	srcdir="."
+	repo_url="${CONF_REPO_URL}"
+	upload_url="${CONF_UPLOAD_URL}/samba/rc/"
+	download_url="${CONF_DOWNLOAD_URL}/samba/rc/"
+
+	verify_fn="verify_samba_rc"
+	check_fn="check_nopatch"
+	upload_fn="upload_nopatch"
+	announcement_fn="announcement_samba_rc"
+	fullcmds="verify create check whatsnew announcement push upload announce"
+	;;
+samba-stable)
+	test -z "${GPG_USER-}" && {
+		GPG_USER='Samba Distribution Verification Key <samba-bugs@samba.org>'
+	}
+
+	test -z "${GPG_KEYID-}" && {
+		GPG_KEYID='AA99442FB680B620'
+	}
+
+	productbase="samba"
+	srcdir="."
+	repo_url="${CONF_REPO_URL}"
+	upload_url="${CONF_UPLOAD_URL}/"
+	download_url="${CONF_DOWNLOAD_URL}/"
+	history_url="${CONF_HISTORY_URL}/samba/history/"
+
+	verify_fn="verify_samba_stable"
+	check_fn="check_samba_stable"
+	upload_fn="upload_samba_stable"
+	announcement_fn="announcement_samba_stable"
+	fullcmds="verify create patch check announcement push upload announce"
+	;;
+TODO-samba-security)
+	test -z "${GPG_USER-}" && {
+		GPG_USER='Samba Distribution Verification Key <samba-bugs@samba.org>'
+	}
+
+	test -z "${GPG_KEYID-}" && {
+		GPG_KEYID='AA99442FB680B620'
+	}
+
+	productbase="samba"
+	srcdir="."
+	repo_url="${CONF_REPO_URL}"
+	upload_url="${CONF_UPLOAD_URL}/"
+	download_url="${CONF_DOWNLOAD_URL}/"
+	history_url="${CONF_HISTORY_URL}/samba/history/"
+
+	verify_fn="verify_samba_stable"
+	check_fn="check_samba_stable"
+	upload_fn="upload_samba_stable"
+	announcement_fn="announcement_samba_security"
+	fullcmds="verify create patch check announcement"
+	next_cmd="push"
+	;;
+*)
+	usage
+	echo "Unknown product ${product}"
+	exit 1
+	;;
+esac
+
+pushd ${srcdir} || {
+	echo "srcdir[${srcdir}] does not exist"
+	exit 1
+}
+
+trap_handler()
+{
+	echo ""
+	echo "ERROR: cleaning up"
+	echo ""
+
+	for t in ${CLEANUP_TAGS}; do
+		echo "Removing tag[${t}]"
+		git tag -v "${t}" && {
+			git tag -d "${t}" || {
+				echo "failed to remove tag ${t}"
+			}
+		}
+	done
+
+	for f in ${CLEANUP_FILES}; do
+		echo "Removing file[${f}]"
+		test -f "${f}" && {
+			rm "${f}" || {
+				echo "failed to remove ${f}"
+			}
+		}
+	done
+
+	for d in ${CLEANUP_DIRS}; do
+		echo "Removing dir[${d}]"
+		test -d "${d}" && {
+			rm -rf "${d}" || {
+				echo "failed to remove ${d}"
+			}
+		}
+	done
+}
+
+CLEANUP_TAGS=""
+CLEANUP_FILES=""
+CLEANUP_DIRS=""
+trap trap_handler INT QUIT TERM EXIT
+
+cmd_allowed "${globalcmd}" fullrelease ${fullcmds} || {
+	usage
+	echo "command[${globalcmd}] not supported for product[${product}]"
+	exit 1
+}
+
+case "${globalcmd}" in
+fullrelease)
+	check_args "${globalcmd}" "$#" "0" || exit 1
+	cmds="${fullcmds}"
+	;;
+create)
+	check_args "${globalcmd}" "$#" "0" || exit 1
+	check_args "create" "$#" "0" || exit 1
+
+	cmds=""
+	cmd_allowed "verify" ${fullcmds} && {
+		cmds="${cmds} verify"
+	}
+	cmds="${cmds} create"
+	cmd_allowed "whatsnew" ${fullcmds} && {
+		cmds="${cmds} whatsnew"
+	}
+	cmd_allowed "patch" ${fullcmds} && {
+		cmds="${cmds} patch"
+	}
+	cmds="${cmds} check"
+	cmd_allowed "announcement" ${fullcmds} && {
+		cmds="${cmds} announcement"
+	}
+	next_cmd="push"
+	;;
+push)
+	check_args "${globalcmd}" "$#" "1" || exit 1
+	tagname="$1"
+	cmds="check push"
+	next_cmd="upload"
+	;;
+upload)
+	check_args "${globalcmd}" "$#" "1" || exit 1
+	tagname="$1"
+	cmds="check upload"
+	cmd_allowed "announce" ${fullcmds} && {
+		next_cmd="announce"
+	}
+	;;
+announce)
+	check_args "${globalcmd}" "$#" "1" || exit 1
+	tagname="$1"
+	cmds="check announce"
+	;;
+*)
+	usage
+	echo "Unknown command ${globalcmd}"
+	exit 1
+	;;
+esac
+
+TMPDIR="release.$$"
+CLEANUP_DIRS="${CLEANUP_DIRS} ${TMPDIR}"
+umask 0077
+mkdir "${TMPDIR}"
+umask 0022
+
+for cmd in ${cmds}; do
+	echo "Starting subcommand[${cmd}]"
+	${cmd}_release || {
+		echo "Failed subcommand[${cmd}]"
+		exit 1
+	}
+	echo "Finished subcommand[${cmd}]"
+done
+
+test -d "${TMPDIR}" && {
+	rm -rf "${TMPDIR}" || {
+		echo "failed to remove ${TMPDIR}"
+	}
+}
+
+test -n "${next_cmd}" && {
+	echo "Continue with '$0 ${product} ${next_cmd} ${tagname}'."
+}
+
+trap - INT QUIT TERM EXIT
+
+exit 0
diff --git a/script/show_test_time b/script/show_test_time
new file mode 100755
index 0000000..70d29d7
--- /dev/null
+++ b/script/show_test_time
@@ -0,0 +1,29 @@
+#!/usr/bin/python
+import optparse
+import os.path
+import subprocess
+import sys
+
+parser = optparse.OptionParser()
+parser.add_option("--limit", dest="limit", type=int,
+                  help="Limit to this number of output entries.", default=0)
+(opts, args) = parser.parse_args()
+
+durations = {}
+
+cmd = "subunit-1to2 | subunit-ls --times --no-passthrough"
+
+p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=sys.stdin, shell=True)
+for l in p.stdout:
+    l = l.strip()
+    (name, duration) = l.rsplit(" ", 1)
+    durations[name] = float(duration)
+
+if opts.limit:
+    print("Top %d tests by run time:" % opts.limit)
+
+for i, (name, length) in enumerate(sorted(
+        durations.items(), key=lambda x: x[1], reverse=True)):
+    if opts.limit and i == opts.limit:
+        break
+    print("%d: %s -> %ds" % (i+1, name, length))
diff --git a/script/show_testsuite_time b/script/show_testsuite_time
new file mode 100755
index 0000000..6e5808a
--- /dev/null
+++ b/script/show_testsuite_time
@@ -0,0 +1,51 @@
+#!/usr/bin/env perl
+use Time::Local ('timegm');
+my $in = STDIN;
+use strict;
+
+my $intest=0;
+my $name;
+my $start=0;
+my $end=0;
+my %hash;
+my $fh;
+my $max=0;
+if ($#ARGV >= 0) {
+	open($fh, "<", $ARGV[0]) || die "can't open ".$ARGV[0];
+} else {
+	$fh = $in;
+}
+if ($#ARGV >= 1) {
+	$max = $ARGV[1];
+	if ($max =~ /\D/) {
+		die "not a decimal number: '$max'";
+	}
+}
+
+print "TOP $max slowest tests\n";
+
+while(<$fh>)
+{
+	if (m/^testsuite: (.*)/) {
+		$intest = 1;
+		$name = $1;
+	}
+	if (m/testsuite-\w+:/) {
+		$hash{"$name -> ".($end - $start)} = $end - $start;
+		$intest = 0;
+		$start = 0;
+	}
+	if (m/^time: (\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)/ && $intest) {
+		my $ts=timegm($6,$5,$4,$3,$2 - 1,$1 - 1900);
+		if ($start == 0) {
+			$start = $ts;
+		} else {
+			$end = $ts;
+		}
+	}
+}
+my @sorted = sort { $hash{$b}<=>$hash{$a} } keys(%hash);
+$max = @sorted if (($max <= 0) or ($max > @sorted));
+for my $l (@sorted[0..($max - 1)]) {
+	print $l."\n";
+}
diff --git a/script/traffic_learner b/script/traffic_learner
new file mode 100755
index 0000000..303956e
--- /dev/null
+++ b/script/traffic_learner
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+# Generate a traffic model from a traffic summary file
+#
+# Copyright (C) Catalyst IT Ltd. 2017
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import sys
+import argparse
+
+sys.path.insert(0, "bin/python")
+from samba.emulate import traffic
+
+from samba.logger import get_samba_logger
+logger = get_samba_logger(name=__name__, level=20)
+error = logger.error
+info = logger.info
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__,
+                        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('-o', '--out',
+                        help="write model here")
+    parser.add_argument('--dns-mode', choices=['inline', 'count'],
+                        help='how to deal with DNS', default='count')
+    parser.add_argument('SUMMARY_FILE', nargs='*', type=argparse.FileType('r'),
+                        default=[sys.stdin],
+                        help="read from this file (default STDIN)")
+    args = parser.parse_args()
+
+    if args.out is None:
+        error("No output file was specified to write the model to.")
+        error("Please specify a filename using the --out option.")
+        return 1
+
+    try:
+        outfile = open(args.out, 'w')
+    except IOError as e:
+        error("could not open %s" % args.out)
+        error(e)
+        return 1
+
+    if args.SUMMARY_FILE is sys.stdin:
+        info("reading from STDIN...")
+
+    (conversations,
+     interval,
+     duration,
+     dns_counts) = traffic.ingest_summaries(args.SUMMARY_FILE,
+                                            dns_mode=args.dns_mode)
+
+    model = traffic.TrafficModel()
+    info("learning model")
+    if args.dns_mode == 'count':
+        model.learn(conversations, dns_counts)
+    else:
+        model.learn(conversations)
+
+    model.save(args.out)
+
+sys.exit(main())
diff --git a/script/traffic_replay b/script/traffic_replay
new file mode 100755
index 0000000..60b7adb
--- /dev/null
+++ b/script/traffic_replay
@@ -0,0 +1,445 @@
+#!/usr/bin/env python3
+# Generates samba network traffic
+#
+# Copyright (C) Catalyst IT Ltd. 2017
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import sys
+import os
+import optparse
+import tempfile
+import shutil
+import random
+
+sys.path.insert(0, "bin/python")
+
+from samba import gensec, get_debug_level
+from samba.emulate import traffic
+import samba.getopt as options
+from samba.logger import get_samba_logger
+from samba.samdb import SamDB
+from samba.auth import system_session
+
+
+def print_err(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+def main():
+
+    desc = ("Generates network traffic 'conversations' based on a model generated"
+            " by script/traffic_learner. This traffic is sent to <dns-hostname>,"
+            " which is the full DNS hostname of the DC being tested.")
+
+    parser = optparse.OptionParser(
+        "%prog [--help|options] <model-file> <dns-hostname>",
+        description=desc)
+
+    parser.add_option('--dns-rate', type='float', default=0,
+                      help='fire extra DNS packets at this rate')
+    parser.add_option('--dns-query-file', dest="dns_query_file",
+                      help='A file contains DNS query list')
+    parser.add_option('-B', '--badpassword-frequency',
+                      type='float', default=0.0,
+                      help='frequency of connections with bad passwords')
+    parser.add_option('-K', '--prefer-kerberos',
+                      action="store_true",
+                      help='prefer kerberos when authenticating test users')
+    parser.add_option('-I', '--instance-id', type='int', default=0,
+                      help='Instance number, when running multiple instances')
+    parser.add_option('-t', '--timing-data',
+                      help=('write individual message timing data here '
+                            '(- for stdout)'))
+    parser.add_option('--preserve-tempdir', default=False, action="store_true",
+                      help='do not delete temporary files')
+    parser.add_option('-F', '--fixed-password',
+                      type='string', default=None,
+                      help=('Password used for the test users created. '
+                            'Required'))
+    parser.add_option('-c', '--clean-up',
+                      action="store_true",
+                      help='Clean up the generated groups and user accounts')
+    parser.add_option('--random-seed', type='int', default=None,
+                      help='Use to keep randomness consistent across multiple runs')
+    parser.add_option('--stop-on-any-error',
+                      action="store_true",
+                      help='abort the whole thing if a child fails')
+    model_group = optparse.OptionGroup(parser, 'Traffic Model Options',
+                                       'These options alter the traffic '
+                                       'generated by the model')
+    model_group.add_option('-S', '--scale-traffic', type='float',
+                           help=('Increase the number of conversations by '
+                                 'this factor (or use -T)'))
+    parser.add_option('-T', '--packets-per-second', type=float,
+                      help=('attempt this many packets per second '
+                            '(alternative to -S)'))
+    parser.add_option('--old-scale',
+                      action="store_true",
+                      help='emulate the old scale for traffic')
+    model_group.add_option('-D', '--duration', type='float', default=60.0,
+                           help=('Run model for this long (approx). '
+                                 'Default 60s for models'))
+    model_group.add_option('--latency-timeout', type='float', default=None,
+                           help=('Wait this long for last packet to finish'))
+    model_group.add_option('-r', '--replay-rate', type='float', default=1.0,
+                           help='Replay the traffic faster by this factor')
+    model_group.add_option('--conversation-persistence', type='float',
+                           default=0.0,
+                           help=('chance (0 to 1) that a conversation waits '
+                                 'when it would have died'))
+    model_group.add_option('--traffic-summary',
+                           help=('Generate a traffic summary file and write '
+                                 'it here (- for stdout)'))
+    parser.add_option_group(model_group)
+
+    user_gen_group = optparse.OptionGroup(parser, 'Generate User Options',
+                                          "Add extra user/groups on the DC to "
+                                          "increase the DB size. These extra "
+                                          "users aren't used for traffic "
+                                          "generation.")
+    user_gen_group.add_option('-G', '--generate-users-only',
+                              action="store_true",
+                              help='Generate the users, but do not replay '
+                              'the traffic')
+    user_gen_group.add_option('-n', '--number-of-users', type='int', default=0,
+                              help='Total number of test users to create')
+    user_gen_group.add_option('--number-of-groups', type='int', default=None,
+                              help='Create this many groups')
+    user_gen_group.add_option('--average-groups-per-user',
+                              type='int', default=0,
+                              help='Assign the test users to this '
+                              'many groups on average')
+    user_gen_group.add_option('--group-memberships', type='int', default=0,
+                              help='Total memberships to assign across all '
+                              'test users and all groups')
+    user_gen_group.add_option('--max-members', type='int', default=None,
+                              help='Max users to add to any one group')
+    parser.add_option_group(user_gen_group)
+
+    sambaopts = options.SambaOptions(parser)
+    parser.add_option_group(sambaopts)
+    parser.add_option_group(options.VersionOptions(parser))
+    credopts = options.CredentialsOptions(parser)
+    parser.add_option_group(credopts)
+
+    # the --no-password credential doesn't make sense for this tool
+    if parser.has_option('-N'):
+        parser.remove_option('-N')
+
+    opts, args = parser.parse_args()
+
+    # First ensure we have reasonable arguments
+
+    if len(args) == 1:
+        model_file = None
+        host    = args[0]
+    elif len(args) == 2:
+        model_file, host = args
+    else:
+        parser.print_usage()
+        return
+
+    lp = sambaopts.get_loadparm()
+    debuglevel = get_debug_level()
+    logger = get_samba_logger(name=__name__,
+                              verbose=debuglevel > 3,
+                              quiet=debuglevel < 1)
+
+    traffic.DEBUG_LEVEL = debuglevel
+    # pass log level down to traffic module to make sure level is controlled
+    traffic.LOGGER.setLevel(logger.getEffectiveLevel())
+
+    if opts.clean_up:
+        logger.info("Removing user and machine accounts")
+        lp    = sambaopts.get_loadparm()
+        creds = credopts.get_credentials(lp)
+        creds.set_gensec_features(creds.get_gensec_features() | gensec.FEATURE_SEAL)
+        ldb   = traffic.openLdb(host, creds, lp)
+        traffic.clean_up_accounts(ldb, opts.instance_id)
+        exit(0)
+
+    if model_file:
+        if not os.path.exists(model_file):
+            logger.error("Model file %s doesn't exist" % model_file)
+            sys.exit(1)
+    # the model-file can be omitted for --generate-users-only and
+    # --cleanup-up, but it should be specified in all other cases
+    elif not opts.generate_users_only:
+        logger.error("No model file specified to replay traffic from")
+        sys.exit(1)
+
+    if not opts.fixed_password:
+        logger.error(("Please use --fixed-password to specify a password"
+                      " for the users created as part of this test"))
+        sys.exit(1)
+
+    if opts.random_seed is not None:
+        random.seed(opts.random_seed)
+
+    creds = credopts.get_credentials(lp)
+    creds.set_gensec_features(creds.get_gensec_features() | gensec.FEATURE_SEAL)
+
+    domain = creds.get_domain()
+    if domain:
+        lp.set("workgroup", domain)
+    else:
+        domain = lp.get("workgroup")
+        if domain == "WORKGROUP":
+            logger.error(("NETBIOS domain does not appear to be "
+                          "specified, use the --workgroup option"))
+            sys.exit(1)
+
+    if not opts.realm and not lp.get('realm'):
+        logger.error("Realm not specified, use the --realm option")
+        sys.exit(1)
+
+    if opts.generate_users_only and not (opts.number_of_users or
+                                         opts.number_of_groups):
+        logger.error(("Please specify the number of users and/or groups "
+                      "to generate."))
+        sys.exit(1)
+
+    if opts.group_memberships and opts.average_groups_per_user:
+        logger.error(("--group-memberships and --average-groups-per-user"
+                      " are incompatible options - use one or the other"))
+        sys.exit(1)
+
+    if not opts.number_of_groups and opts.average_groups_per_user:
+        logger.error(("--average-groups-per-user requires "
+                      "--number-of-groups"))
+        sys.exit(1)
+
+    if opts.number_of_groups and opts.average_groups_per_user:
+        if opts.number_of_groups < opts.average_groups_per_user:
+            logger.error(("--average-groups-per-user can not be more than "
+                          "--number-of-groups"))
+            sys.exit(1)
+
+    if not opts.number_of_groups and opts.group_memberships:
+        logger.error("--group-memberships requires --number-of-groups")
+        sys.exit(1)
+
+    if opts.scale_traffic is not None and opts.packets_per_second is not None:
+        logger.error("--scale-traffic and --packets-per-second "
+                     "are incompatible. Use one or the other.")
+        sys.exit(1)
+
+    if not opts.scale_traffic and not opts.packets_per_second:
+        logger.info("No packet rate specified. Using --scale-traffic=1.0")
+        opts.scale_traffic = 1.0
+
+    if opts.timing_data not in ('-', None):
+        try:
+            open(opts.timing_data, 'w').close()
+        except IOError:
+            # exception info will be added to log automatically
+            logger.exception(("the supplied timing data destination "
+                              "(%s) is not writable" % opts.timing_data))
+            sys.exit()
+
+    if opts.traffic_summary not in ('-', None):
+        try:
+            open(opts.traffic_summary, 'w').close()
+        except IOError:
+            # exception info will be added to log automatically
+            if debuglevel > 0:
+                import traceback
+                traceback.print_exc()
+            logger.exception(("the supplied traffic summary destination "
+                              "(%s) is not writable" % opts.traffic_summary))
+            sys.exit()
+
+    if opts.old_scale:
+        # we used to use a silly calculation based on the number
+        # of conversations; now we use the number of packets and
+        # scale traffic accurately. To roughly compare with older
+        # numbers you use --old-scale which approximates as follows:
+        opts.scale_traffic *= 0.55
+
+    # ingest the model
+    if model_file and not opts.generate_users_only:
+        model = traffic.TrafficModel()
+        try:
+            model.load(model_file)
+        except ValueError:
+            if debuglevel > 0:
+                import traceback
+                traceback.print_exc()
+            logger.error(("Could not parse %s, which does not seem to be "
+                          "a model generated by script/traffic_learner."
+                          % model_file))
+            sys.exit(1)
+
+        logger.info(("Using the specified model file to "
+                     "generate conversations"))
+
+        if opts.scale_traffic:
+            packets_per_second = model.scale_to_packet_rate(opts.scale_traffic)
+        else:
+            packets_per_second =  opts.packets_per_second
+
+        conversations = \
+            model.generate_conversation_sequences(
+                packets_per_second,
+                opts.duration,
+                opts.replay_rate,
+                opts.conversation_persistence)
+    else:
+        conversations = []
+
+    if opts.number_of_users and opts.number_of_users < len(conversations):
+        logger.error(("--number-of-users (%d) is less than the "
+                      "number of conversations to replay (%d)"
+                     % (opts.number_of_users, len(conversations))))
+        sys.exit(1)
+
+    number_of_users = max(opts.number_of_users, len(conversations))
+
+    if opts.number_of_groups is None:
+        opts.number_of_groups = max(int(number_of_users / 10), 1)
+
+    max_memberships = number_of_users * opts.number_of_groups
+
+    if not opts.group_memberships and opts.average_groups_per_user:
+        opts.group_memberships = opts.average_groups_per_user * number_of_users
+        logger.info(("Using %d group-memberships based on %u average "
+                     "memberships for %d users"
+                     % (opts.group_memberships,
+                        opts.average_groups_per_user, number_of_users)))
+
+    if opts.group_memberships > max_memberships:
+        logger.error(("The group memberships specified (%d) exceeds "
+                      "the total users (%d) * total groups (%d)"
+                      % (opts.group_memberships, number_of_users,
+                         opts.number_of_groups)))
+        sys.exit(1)
+
+    # if no groups were specified by the user, then make sure we create some
+    # group memberships (otherwise it's not really a fair test)
+    if not opts.group_memberships and not opts.average_groups_per_user:
+        opts.group_memberships = min(number_of_users * 5, max_memberships)
+
+    # Get an LDB connection.
+    try:
+        # if we're only adding users, then it's OK to pass a sam.ldb filepath
+        # as the host, which creates the users much faster. In all other cases
+        # we should be connecting to a remote DC
+        if opts.generate_users_only and os.path.isfile(host):
+            ldb = SamDB(url="ldb://{0}".format(host),
+                        session_info=system_session(), lp=lp)
+        else:
+            ldb = traffic.openLdb(host, creds, lp)
+    except:
+        logger.error(("\nInitial LDAP connection failed! Did you supply "
+                      "a DNS host name and the correct credentials?"))
+        sys.exit(1)
+
+    if opts.generate_users_only:
+        # generate computer accounts for added realism. Assume there will be
+        # some overhang with more computer accounts than users
+        computer_accounts = int(1.25 * number_of_users)
+        traffic.generate_users_and_groups(ldb,
+                                          opts.instance_id,
+                                          opts.fixed_password,
+                                          opts.number_of_users,
+                                          opts.number_of_groups,
+                                          opts.group_memberships,
+                                          opts.max_members,
+                                          machine_accounts=computer_accounts,
+                                          traffic_accounts=False)
+        sys.exit()
+
+    tempdir = tempfile.mkdtemp(prefix="samba_tg_")
+    logger.info("Using temp dir %s" % tempdir)
+
+    traffic.generate_users_and_groups(ldb,
+                                      opts.instance_id,
+                                      opts.fixed_password,
+                                      number_of_users,
+                                      opts.number_of_groups,
+                                      opts.group_memberships,
+                                      opts.max_members,
+                                      machine_accounts=len(conversations),
+                                      traffic_accounts=True)
+
+    accounts = traffic.generate_replay_accounts(ldb,
+                                                opts.instance_id,
+                                                len(conversations),
+                                                opts.fixed_password)
+
+    statsdir = traffic.mk_masked_dir(tempdir, 'stats')
+
+    if opts.traffic_summary:
+        if opts.traffic_summary == '-':
+            summary_dest = sys.stdout
+        else:
+            summary_dest = open(opts.traffic_summary, 'w')
+
+        logger.info("Writing traffic summary")
+        summaries = []
+        for c in traffic.seq_to_conversations(conversations):
+            summaries += c.replay_as_summary_lines()
+
+        summaries.sort()
+        for (time, line) in summaries:
+            print(line, file=summary_dest)
+
+        exit(0)
+
+    traffic.replay(conversations,
+                   host,
+                   lp=lp,
+                   creds=creds,
+                   accounts=accounts,
+                   dns_rate=opts.dns_rate,
+                   dns_query_file=opts.dns_query_file,
+                   duration=opts.duration,
+                   latency_timeout=opts.latency_timeout,
+                   badpassword_frequency=opts.badpassword_frequency,
+                   prefer_kerberos=opts.prefer_kerberos,
+                   statsdir=statsdir,
+                   domain=domain,
+                   base_dn=ldb.domain_dn(),
+                   ou=traffic.ou_name(ldb, opts.instance_id),
+                   tempdir=tempdir,
+                   stop_on_any_error=opts.stop_on_any_error,
+                   domain_sid=ldb.get_domain_sid(),
+                   instance_id=opts.instance_id)
+
+    if opts.timing_data == '-':
+        timing_dest = sys.stdout
+    elif opts.timing_data is None:
+        timing_dest = None
+    else:
+        timing_dest = open(opts.timing_data, 'w')
+
+    logger.info("Generating statistics")
+    traffic.generate_stats(statsdir, timing_dest)
+
+    if not opts.preserve_tempdir:
+        logger.info("Removing temporary directory")
+        shutil.rmtree(tempdir)
+    else:
+        # delete the empty directories anyway. There are thousands of
+        # them and they're EMPTY.
+        for d in os.listdir(tempdir):
+            if d.startswith('conversation-'):
+                path = os.path.join(tempdir, d)
+                try:
+                    os.rmdir(path)
+                except OSError as e:
+                    logger.info("not removing %s (%s)" % (path, e))
+
+main()
diff --git a/script/traffic_summary.pl b/script/traffic_summary.pl
new file mode 100755
index 0000000..295a320
--- /dev/null
+++ b/script/traffic_summary.pl
@@ -0,0 +1,707 @@
+#! /usr/bin/perl
+#
+# Summarise tshark pdml output into a form suitable for the load test tool
+#
+# Copyright (C) Catalyst.Net Ltd 2017
+#
+# Catalyst.Net's contribution was written by Gary Lockyer
+# <gary@catalyst.net.nz>.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+use warnings;
+use strict;
+
+use Getopt::Long;
+use Pod::Usage;
+
+BEGIN {
+    unless (eval "require XML::Twig") {
+        warn "traffic_summary requires the perl module XML::Twig\n" .
+             "on Ubuntu/Debian releases run\n".
+             "  sudo apt install libxml-twig-perl \n".
+             "or install from CPAN\n".
+             "\nThe reported error was:\n$@";
+        exit(1);
+    }
+}
+
+
+my %ip_map;              # Map of IP address to sequence number
+my $ip_sequence = 0;     # count of unique IP addresses seen
+
+
+my $timestamp;           # Packet timestamp
+my $stream;              # Wireshark stream number
+my $ip_proto;            # IP protocol (IANA protocol number)
+my $source;              # source IP address
+my $dest;                # destination address
+my $proto;               # application protocol name
+my $description;         # protocol specific description
+my %proto_data;          # protocol specific data captured for the current packet
+my $malformed_packet;    # Indicates the current packet has errors
+my $ldap_filter;         # cleaned ldap filter
+my $ldap_attributes;     # attributes requested in an ldap query
+
+
+
+# Dispatch table mapping the wireshark variables of interest to the
+# functions responsible for processing them
+my %field_dispatch_table = (
+    'timestamp'                       => \&timestamp,
+    'ip.src'                          => \&ip_src,
+    'ipv6.src'                        => \&ip_src,
+    'ip.dst'                          => \&ip_dst,
+    'ipv6.dst'                        => \&ip_dst,
+    'ip.proto'                        => \&ip_proto,
+    'udp.stream'                      => \&stream,
+    'tcp.stream'                      => \&stream,
+    'dns.flags.opcode'                => \&field_data,
+    'dns.flags.response'              => \&field_data,
+    'netlogon.opnum'                  => \&field_data,
+    'kerberos.msg_type'               => \&field_data,
+    'smb.cmd'                         => \&field_data,
+    'smb2.cmd'                        => \&field_data,
+    'ldap.protocolOp'                 => \&field_data,
+    'gss-api.OID'                     => \&field_data,
+    'ldap.gssapi_encrypted_payload'   => \&field_data,
+    'ldap.baseObject'                 => \&field_data,
+    'ldap.scope'                      => \&field_data,
+    'ldap.AttributeDescription'       => \&ldap_attribute,
+    'ldap.modification_element'       => \&ldap_add_modify,
+    'ldap.AttributeList_item_element' => \&ldap_add_modify,
+    'ldap.operation'                  => \&field_data,
+    'ldap.authentication'             => \&field_data,
+    'lsarpc.opnum'                    => \&field_data,
+    'samr.opnum'                      => \&field_data,
+    'dcerpc.pkt_type'                 => \&field_data,
+    'epm.opnum'                       => \&field_data,
+    'dnsserver.opnum'                 => \&field_data,
+    'drsuapi.opnum'                   => \&field_data,
+    'browser.command'                 => \&field_data,
+    'smb_netlogon.command'            => \&field_data,
+    'srvsvc.opnum'                    => \&field_data,
+    'nbns.flags.opcode'               => \&field_data,
+    'nbns.flags.response'             => \&field_data,
+    '_ws.expert.message'              => \&field_data,
+);
+
+# Dispatch table mapping protocols to the routine responsible for formatting
+# their output.  Protocols not in this table are ignored.
+#
+my %proto_dispatch_table = (
+    'dns'          => sub { return format_opcode( 'dns.flags.response')},
+    'rpc_netlogon' => sub { return format_opcode( 'netlogon.opnum')},
+    'kerberos'     => \&format_kerberos,
+    'smb'          => sub { return format_opcode( 'smb.cmd')},
+    'smb2'         => sub { return format_opcode( 'smb2.cmd')},
+    'ldap'         => \&format_ldap,
+    'cldap'        => \&format_ldap,
+    'lsarpc'       => sub { return format_opcode( 'lsarpc.opnum')},
+    'samr'         => sub { return format_opcode( 'samr.opnum')},
+    'dcerpc'       => sub { return format_opcode( 'dcerpc.pkt_type')},
+    'epm'          => sub { return format_opcode( 'epm.opnum')},
+    'dnsserver'    => sub { return format_opcode( 'dnsserver.opnum')},
+    'drsuapi'      => sub { return format_opcode( 'drsuapi.opnum')},
+    'browser'      => sub { return format_opcode( 'browser.command')},
+    'smb_netlogon' => sub { return format_opcode( 'smb_netlogon.command')},
+    'srvsvc'       => sub { return format_opcode( 'srvsvc.opnum')},
+    'nbns'         => sub { return format_opcode( 'nbns.flags.response')},
+);
+
+# XPath entry to extract the kerberos cname
+my $kerberos_cname_path =
+      'packet/proto/field[@name = "kerberos.as_req_element"]'
+    . '/field[@name = "kerberos.req_body_element"]'
+    . '/field[@name = "kerberos.cname_element"]'
+    . '/field[@name = "kerberos.name_string"]'
+    . '/field[@name = "kerberos.KerberosString"]';
+
+# XPath entry to extract the ldap filter
+my $ldap_filter_path =
+      'field[@name = "ldap.searchRequest_element"]/field';
+
+
+# Create an XML Twig parser and register the event handlers.
+#
+my $t = XML::Twig->new(
+    start_tag_handlers => {
+        'packet'       => \&packet_start,
+    },
+    twig_handlers => {
+         'packet'              => \&packet,
+         'proto'               => \&protocol,
+         'field'               => \&field,
+         $kerberos_cname_path  => \&kerberos_cname,
+         $ldap_filter_path     => \&ldap_filter,
+    },
+);
+
+#------------------------------------------------------------------------------
+# Main loop
+#
+#------------------------------------------------------------------------------
+my $help = 0;
+GetOptions( 'help|h' => \$help) or pod2usage(2);
+pod2usage(1) if $help;
+
+if (@ARGV) {
+    foreach my $file (@ARGV) {
+        eval {
+            $t->parsefile( $file);
+        };
+        if ($@) {
+            print STDERR "Unable to process $file, ".
+                         "did you run tshark with the -T pdml option?";
+        }
+    }
+} else {
+    pod2usage(1) if -t STDIN;
+    eval {
+        $t->parse( \*STDIN);
+    };
+    if ($@) {
+        print STDERR "Unable to process input, ".
+                     "are you running tshark with the -T pdml option?";
+    }
+}
+
+
+#------------------------------------------------------------------------------
+# New packet detected reset the globals
+#------------------------------------------------------------------------------
+sub packet_start
+{
+    my ($t, $packet) = @_;
+    $timestamp           = "";
+    $stream              = "";
+    $ip_proto            = "";
+    $source              = "";
+    $dest                = "";
+    $description         = undef;
+    %proto_data          = ();
+    $malformed_packet    = undef;
+    $ldap_filter         = "";
+    $ldap_attributes     = "";
+}
+
+#------------------------------------------------------------------------------
+# Complete packet element parsed from the XML feed
+# output the protocol summary if required
+#------------------------------------------------------------------------------
+sub packet
+{
+    my ($t, $packet) = @_;
+
+    my $data;
+    if (exists $proto_dispatch_table{$proto}) {
+        if ($malformed_packet) {
+            $data = "\t\t** Malformed Packet ** " . ($proto_data{'_ws.expert.message.show'} || '');
+        } else {
+            my $rsub = $proto_dispatch_table{$proto};
+            $data = &$rsub();
+        }
+        print "$timestamp\t$ip_proto\t$stream\t$source\t$dest\t$proto\t$data\n";
+    }
+    $t->purge;
+}
+
+#------------------------------------------------------------------------------
+# Complete protocol element parsed from the XML input
+# Update the protocol name
+#------------------------------------------------------------------------------
+sub protocol
+{
+    my ($t, $protocol) = @_;
+    if ($protocol->{att}->{showname}) {
+    }
+    # Tag a packet as malformed if the protocol is _ws.malformed
+    # and the hide attribute is not 'yes'
+    if ($protocol->{att}->{name} eq '_ws.malformed'
+        && !($protocol->{att}->{hide} && $protocol->{att}->{hide} eq 'yes')
+    ) {
+        $malformed_packet = 1;
+    }
+    # Don't set the protocol name if it's a wireshark malformed
+    # protocol entry, or the packet was truncated during capture
+    my $p = $protocol->{att}->{name};
+    if ($p ne '_ws.malformed' && $p ne '_ws.short')  {
+        $proto = $p;
+    }
+}
+
+
+#------------------------------------------------------------------------------
+# Complete field element parsed, extract any data of interest
+#------------------------------------------------------------------------------
+sub field
+{
+    my ($t, $field) = @_;
+    my $name = $field->{att}->{name};
+
+    # Only process the field if it has a corresponding entry in
+    # %field_dispatch_table
+    if (exists $field_dispatch_table{$name}) {
+        my $rsub = $field_dispatch_table{$name};
+        &$rsub( $field);
+    }
+}
+
+#------------------------------------------------------------------------------
+# Process a timestamp field element
+#------------------------------------------------------------------------------
+sub timestamp
+{
+    my ($field) = @_;
+    $timestamp = $field->{att}->{value};
+}
+
+#------------------------------------------------------------------------------
+# Process a wireshark stream element, used to group a sequence of requests
+# and responses between two IP addresses
+#------------------------------------------------------------------------------
+sub stream
+{
+    my ($field) = @_;
+    $stream = $field->{att}->{show};
+}
+
+#------------------------------------------------------------------------------
+# Process a source ip address field, mapping the IP address to it's
+# corresponding sequence number.
+#------------------------------------------------------------------------------
+sub ip_src
+{
+    my ($field) = @_;
+    $source = map_ip( $field);
+}
+
+#------------------------------------------------------------------------------
+# Process a destination ip address field, mapping the IP address to it's
+# corresponding sequence number.
+#------------------------------------------------------------------------------
+sub ip_dst
+{
+    my ($field) = @_;
+    $dest = map_ip( $field);
+}
+
+#------------------------------------------------------------------------------
+# Process an ip protocol element, extracting IANA protocol number
+#------------------------------------------------------------------------------
+sub ip_proto
+{
+    my ($field) = @_;
+    $ip_proto = $field->{att}->{value};
+}
+
+
+
+#------------------------------------------------------------------------------
+# Extract an ldap attribute and append it to ldap_attributes
+#------------------------------------------------------------------------------
+sub ldap_attribute
+{
+    my ($field) = @_;
+    my $attribute =  $field->{att}->{show};
+
+    if (defined $attribute) {
+        $ldap_attributes .= "," if $ldap_attributes;
+        $ldap_attributes .= $attribute;
+    }
+}
+
+#------------------------------------------------------------------------------
+# Process a field element, extract the value, show and showname attributes
+# and store them in the %proto_data hash.
+#
+#------------------------------------------------------------------------------
+sub field_data
+{
+    my ($field) = @_;
+    my $name  = $field->{att}->{name};
+    $proto_data{$name.'.value'}    = $field->{att}->{value};
+    $proto_data{$name.'.show'}     = $field->{att}->{show};
+    $proto_data{$name.'.showname'} = $field->{att}->{showname};
+}
+
+#------------------------------------------------------------------------------
+# Process a kerberos cname element, if the cname ends with a $ it's a machine
+# name. Otherwise it's a user name.
+#
+#------------------------------------------------------------------------------
+sub kerberos_cname
+{
+    my ($t, $field) = @_;
+    my $cname =  $field->{att}->{show};
+    my  $type;
+    if( $cname =~ /\$$/) {
+        $type = 'machine';
+     } else {
+        $type = 'user';
+     }
+     $proto_data{'kerberos.cname.type'} = $type;
+}
+
+
+#------------------------------------------------------------------------------
+# Process an ldap filter, remove the values but keep the attribute names
+#------------------------------------------------------------------------------
+sub ldap_filter
+{
+    my ($t, $field) = @_;
+    if ( $field->{att}->{show} && $field->{att}->{show} =~ /^Filter:/) {
+        my $filter = $field->{att}->{show};
+
+        # extract and save the objectClass to keep the value
+        my @object_classes;
+        while ( $filter =~ m/\((objectClass=.*?)\)/g) {
+           push  @object_classes, $1;
+        }
+
+        # extract and save objectCategory and the top level value
+        my @object_categories;
+        while ( $filter =~ m/(\(objectCategory=.*?,|\(objectCategory=.*?\))/g
+        ) {
+            push @object_categories, $1;
+        }
+
+        # Remove all the values from the attributes
+        # Input
+        #     Filter: (nCName=DC=DomainDnsZones,DC=sub1,DC=ad,DC=rh,DC=at,DC=net)
+        # Output
+        #     (nCName)
+        $filter =~ s/^Filter:\s*//; # Remove the 'Filter: ' prefix
+        $filter =~ s/=.*?\)/\)/g;   # Remove from the = to the first )
+
+        # Now restore the parts of objectClass and objectCategory that are being
+        # retained
+        #
+        for my $cat (@object_categories) {
+            $filter =~ s/\(objectCategory\)/$cat/;
+        }
+
+        for my $class (@object_classes) {
+            $filter =~ s/\(objectClass\)/($class)/;
+        }
+
+        $ldap_filter = $filter;
+    } else {
+        # Ok not an ldap filter so call the default field handler
+        field( $t, $field);
+    }
+}
+
+
+#------------------------------------------------------------------------------
+# Extract the attributes from ldap modification and add requests
+#------------------------------------------------------------------------------
+sub ldap_add_modify
+{
+    my ($field) = @_;
+    my $type      = $field->first_child('field[@name="ldap.type"]');
+    my $attribute = $type->{att}->{show} if $type;
+    if (defined $attribute) {
+        $ldap_attributes .= "," if $ldap_attributes;
+        $ldap_attributes .= $attribute;
+    }
+}
+#------------------------------------------------------------------------------
+# Map an IP address to a unique sequence number. Assigning it a sequence number
+# if one has not already been assigned.
+#
+#------------------------------------------------------------------------------
+sub map_ip
+{
+    my ($field) = @_;
+    my $ip = $field->{att}->{show};
+    if ( !exists( $ip_map{$ip})) {
+        $ip_sequence++;
+        $ip_map{$ip} = $ip_sequence;
+    }
+    return $ip_map{$ip};
+}
+
+#------------------------------------------------------------------------------
+# Format a protocol operation code for output.
+#
+#------------------------------------------------------------------------------
+sub format_opcode
+{
+    my ($name) = @_;
+    my $operation   = $proto_data{$name.'.show'};
+    my $description = $proto_data{$name.'.showname'} || '';
+
+    # Strip off the common prefix text, and the trailing (n).
+    # This tidies up most but not all descriptions.
+    $description =~ s/^[^:]*?: ?//     if $description;
+    $description =~ s/^Message is a // if $description;
+    $description =~ s/\(\d+\)\s*$//    if $description;
+    $description =~ s/\s*$//           if $description;
+
+    return "$operation\t$description";
+}
+
+#------------------------------------------------------------------------------
+# Format ldap protocol details for output
+#------------------------------------------------------------------------------
+sub format_ldap
+{
+    my ($name) = @_;
+    if (   exists( $proto_data{'ldap.protocolOp.show'})
+        || exists( $proto_data{'gss-api.OID.show'})
+    ) {
+        my $operation   = $proto_data{'ldap.protocolOp.show'};
+        my $description = $proto_data{'ldap.protocolOp.showname'} || '';
+        my $oid         = $proto_data{'gss-api.OID.show'}         || '';
+        my $base_object = $proto_data{'ldap.baseObject.show'}     || '';
+        my $scope       = $proto_data{'ldap.scope.show'}          || '';
+
+        # Now extract operation specific data
+        my $extra;
+        my $extra_desc;
+        $operation = '' if !defined $operation;
+        if ($operation eq 6) {
+            # Modify operation
+            $extra         = $proto_data{'ldap.operation.show'};
+            $extra_desc    = $proto_data{'ldap.operation.showname'};
+        } elsif ($operation eq 0) {
+            # Bind operation
+            $extra         = $proto_data{'ldap.authentication.show'};
+            $extra_desc    = $proto_data{'ldap.authentication.showname'};
+        }
+        $extra      = '' if !defined $extra;
+        $extra_desc = '' if !defined $extra_desc;
+
+
+        # strip the values out of the base object
+        if ($base_object) {
+            $base_object =~ s/^<//;       # leading '<' if present
+            $base_object =~ s/>$//;       # trailing '>' if present
+            $base_object =~ s/=.*?,/,/g;  # from = up to the next comma
+            $base_object =~ s/=.*?$//;    # from = up to the end of string
+        }
+
+        # strip off the leading prefix on the extra_description
+        # and the trailing (n);
+        $extra_desc =~ s/^[^:]*?: ?//  if $extra_desc;
+        $extra_desc =~ s/\(\d+\)\s*$// if $extra_desc;
+        $extra_desc =~ s/\s*$//        if $extra_desc;
+
+        # strip off the common prefix on the description
+        # and the trailing (n);
+        $description =~ s/^[^:]*?: ?//  if $description;
+        $description =~ s/\(\d+\)\s*$// if $description;
+        $description =~ s/\s*$//        if $description;
+
+        return "$operation\t$description\t$scope\t$base_object"
+              ."\t$ldap_filter\t$ldap_attributes\t$extra\t$extra_desc\t$oid";
+    } else {
+        return "\t*** Unknown ***";
+    }
+}
+
+#------------------------------------------------------------------------------
+# Format kerberos protocol details for output.
+#------------------------------------------------------------------------------
+sub format_kerberos
+{
+    my $msg_type    = $proto_data{'kerberos.msg_type.show'} || '';
+    my $cname_type  = $proto_data{'kerberos.cname.type'} || '';
+    my $description = $proto_data{'kerberos.msg_type.showname'} || '';
+
+    # Tidy up the description
+    $description =~ s/^[^:]*?: ?//  if $description;
+    $description =~ s/\(\d+\)\s*$// if $description;
+    $description =~ s/\s*$//        if $description;
+    return "$msg_type\t$description\t$cname_type";
+}
+
+=pod
+
+=head1 NAME
+
+traffic_summary.pl - summarise tshark pdml output
+
+=head1 USAGE
+
+B<traffic_summary.pl> [FILE...]
+
+Summarise samba network traffic from tshark pdml output.  Produces a tsv
+delimited summary of samba activity.
+
+To process unencrypted traffic
+
+ tshark -r capture.file -T pdml | traffic_summary.pl
+
+To process encrypted kerberos traffic
+
+ tshark -r capture.file -K krb5.keytab -o kerberos.decrypt:true -T pdml | traffic_summary.pl
+
+To display more detailed documentation, including details of the output format
+
+ perldoc traffic_summary.pl
+
+ NOTE: tshark pdml output is very verbose, so it's better to pipe the tshark
+       output directly to traffic_summary, rather than generating
+       intermediate pdml format files.
+
+=head1 OPTIONS
+ B<--help> Display usage message and exit.
+
+=head1 DESCRIPTION
+
+Summarises tshark pdml output into a format suitable for load analysis
+and input into load generation tools.
+
+It reads the pdml input from stdin or the list of files passed on the command line.
+
+
+=head2 Output format
+    The output is tab delimited fields and one line per summarised packet.
+
+=head3 Fields
+     B<timestamp>                Packet timestamp
+     B<IP protocol>              The IANA protocol number
+     B<Wireshark Stream Number>  Calculated by wireshark groups related requests and responses
+     B<Source IP>                The unique sequence number for the source IP address
+     B<Destination IP>           The unique sequence number for the destination IP address
+     B<protocol>                 The protocol name
+     B<opcode>                   The protocol operation code
+     B<Description>              The protocol or operation description
+     B<extra>                    Extra protocol specific data, may be more than one field
+
+
+=head2 IP address mapping
+    Rather than capturing and printing the IP addresses. Each unique IP address
+    seen is assigned a sequence number. So the first IP address seen will be 1,
+    the second 2 ...
+
+=head2 Packets collected
+    Packets containing the following protocol records are summarised:
+      dns
+      rpc_netlogon
+      kerberos
+      smb
+      smb2
+      ldap
+      cldap
+      lsarpc
+      samr
+      dcerpc
+      epm
+      dnsserver
+      drsuapi
+      browser
+      smb_netlogon
+      srvsvc
+      nbns
+
+   Any other packets are ignored.
+
+   In addition to the standard elements extra data is returned for the following
+   protocol record.
+
+=head3 kerberos
+     cname_type  machine cname ends with a $
+                 user    cname does not end with a $
+
+=head3 ldap
+
+     scope                Query Scope
+                            0 - Base
+                            1 - One level
+                            2 - sub tree
+     base_object          ldap base object
+     ldap_filter          the ldap filter, attribute names are retained but the values
+                          are removed.
+     ldap_attributes      ldap attributes, only the names are retained any values are
+                          discarded, with the following two exceptions
+                            objectClass    all the attribute values are retained
+                            objectCategory the top level value is retained
+                                           i.e. everything from the = to the first ,
+
+=head3 ldap modifiyRequest
+     In addition to the standard ldap fields the modification type is also captured
+
+     modify_operator      for modifyRequests this contains the modify operation
+                            0 - add
+                            1 - delete
+                            2 - replace
+     modify_description   a description of the operation if available
+
+=head3 modify bindRequest
+     In addition to the standard ldap fields details of the authentication
+     type are captured
+
+     authentication type  0 - Simple
+                          3 - SASL
+     description          Description of the authentication mechanism
+     oid                  GSS-API OID's
+                            1.2.840.113554.1.2.2   - Kerberos v5
+                            1.2.840.48018.1.2.2    - Kerberos V5
+                               (incorrect, used by old Windows versions)
+                            1.3.6.1.5.5.2          - SPNEGO
+                            1.3.6.1.5.2.5          - IAKERB
+                            1.3.6.1.4.1.311.2.2.10 - NTLM SSP
+                            1.3.6.1.5.5.14         - SCRAM-SHA-1
+                            1.3.6.1.5.5.18         - SCRAM-SHA-256
+                            1.3.6.1.5.5.15.1.1.*   - GSS-EAP
+                            1.3.6.1.5.2.7          - PKU2U
+                            1.3.6.1.5.5.1.1        - SPKM-1
+                            1.3.6.1.5.5.1.2        - SPKM-2
+                            1.3.6.1.5.5.1.3        - SPKM-3
+                            1.3.6.1.5.5.9          - LIPKEY
+                            1.2.752.43.14.2        - NETLOGON
+
+=head1 DEPENDENCIES
+tshark
+XML::Twig         For Ubuntu libxml-twig-perl, or from CPAN
+use Getopt::Long
+use Pod::Usage
+
+
+=head1 Diagnostics
+
+=head2 ** Unknown **
+Unable to determine the operation being performed, for ldap it typically
+indicates a kerberos encrypted operation.
+
+=head2 ** Malformed Packet **
+tshark indicated that the packet was malformed, for ldap it usually indicates TLS
+encrypted traffic.
+
+=head1 LICENSE AND COPYRIGHT
+
+ Copyright (C) Catalyst.Net Ltd 2017
+
+ Catalyst.Net's contribution was written by Gary Lockyer
+ <gary@catalyst.net.nz>.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+=cut