diff options
Diffstat (limited to 'script')
| -rwxr-xr-x | script/attr_count_read | 197 | ||||
| -rwxr-xr-x | script/autobuild.py | 1902 | ||||
| -rwxr-xr-x | script/bisect-test.py | 101 | ||||
| -rwxr-xr-x | script/check-shell-scripts.sh | 20 | ||||
| -rwxr-xr-x | script/clean-source-tree.sh | 33 | ||||
| -rwxr-xr-x | script/codespell.sh | 21 | ||||
| -rwxr-xr-x | script/commit_mark.sh | 21 | ||||
| -rwxr-xr-x | script/compare_cc_results.py | 71 | ||||
| -rwxr-xr-x | script/configure_check_unused.pl | 124 | ||||
| -rwxr-xr-x | script/ctdb-import.msg-filter.sh | 11 | ||||
| -rwxr-xr-x | script/ctdb-import.tree-filter.sh | 13 | ||||
| -rw-r--r-- | script/ctdb-import.txt | 5 | ||||
| -rwxr-xr-x | script/find_python.sh | 9 | ||||
| -rwxr-xr-x | script/findstatic.pl | 70 | ||||
| -rw-r--r-- | script/generate_param.py | 431 | ||||
| -rwxr-xr-x | script/git-hooks/check-trailing-whitespace | 17 | ||||
| -rwxr-xr-x | script/git-hooks/pre-commit-hook | 17 | ||||
| -rwxr-xr-x | script/git-hooks/pre-commit-script | 19 | ||||
| -rwxr-xr-x | script/identity_cc.sh | 10 | ||||
| -rwxr-xr-x | script/release.sh | 1275 | ||||
| -rwxr-xr-x | script/show_test_time | 29 | ||||
| -rwxr-xr-x | script/show_testsuite_time | 51 | ||||
| -rwxr-xr-x | script/traffic_learner | 72 | ||||
| -rwxr-xr-x | script/traffic_replay | 445 | ||||
| -rwxr-xr-x | script/traffic_summary.pl | 707 |
25 files changed, 5671 insertions, 0 deletions
diff --git a/script/attr_count_read b/script/attr_count_read new file mode 100755 index 0000000..2f6a4d0 --- /dev/null +++ b/script/attr_count_read @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +# +# Copyright (C) Catalyst IT Ltd. 2019 +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import sys +import argparse +import struct +import os +from collections import OrderedDict, Counter +from pprint import pprint + +sys.path.insert(0, "bin/python") +import tdb + + +def unpack_uint(filename, casefold=True): + db = tdb.Tdb(filename) + d = {} + for k in db: + v = struct.unpack("I", db[k])[0] + k2 = k.decode('utf-8') + if casefold: + k2 = k2.lower() + if k2 in d: # because casefold + d[k2] += v + else: + d[k2] = v + return d + + +def unpack_ssize_t_pair(filename, casefold): + db = tdb.Tdb(filename) + pairs = [] + for k in db: + key = struct.unpack("nn", k) + v = struct.unpack("I", db[k])[0] + pairs.append((v, key)) + + pairs.sort(reverse=True) + #print(pairs) + return [(k, v) for (v, k) in pairs] + + +DATABASES = [ + ('requested', "debug/attr_counts_requested.tdb", unpack_uint, + "The attribute was specifically requested."), + ('duplicates', "debug/attr_counts_duplicates.tdb", unpack_uint, + "Requested more than once in the same request."), + ('empty request', "debug/attr_counts_empty_req.tdb", unpack_uint, + "No attributes were requested, but these were returned"), + ('null request', "debug/attr_counts_null_req.tdb", unpack_uint, + "The attribute list was NULL and these were returned."), + ('found', "debug/attr_counts_found.tdb", unpack_uint, + "The attribute was specifically requested and it was found."), + ('not found', "debug/attr_counts_not_found.tdb", unpack_uint, + "The attribute was specifically requested but was not found."), + ('unwanted', "debug/attr_counts_unwanted.tdb", unpack_uint, + "The attribute was not requested and it was found."), + ('star match', "debug/attr_counts_star_match.tdb", unpack_uint, + 'The attribute was not specifically requested but "*" was.'), + ('req vs found', "debug/attr_counts_req_vs_found.tdb", unpack_ssize_t_pair, + "How many attributes were requested versus how many were returned."), +] + + +def plot_pair_data(name, data, doc, lim=90): + # Note we keep the matplotlib import internal to this function for + # two reasons: + # 1. Some people won't have matplotlib, but might want to run the + # script. + # 2. The import takes hundreds of milliseconds, which is a + # nuisance if you don't want graphs. + # + # This plot could be improved! + import matplotlib.pylab as plt + fig, ax = plt.subplots() + if lim: + data2 = [] + for p, c in data: + if p[0] > lim or p[1] > lim: + print("not plotting %s: %s" % (p, c)) + continue + data2.append((p, c)) + skipped = len(data) - len(data2) + if skipped: + name += " (excluding %d out of range values)" % skipped + data = data2 + xy, counts = zip(*data) + x, y = zip(*xy) + bins_x = max(x) + 4 + bins_y = max(y) + ax.set_title(name) + ax.scatter(x, y, c=counts) + plt.show() + + +def print_pair_data(name, data, doc): + print(name) + print(doc) + t = "%14s | %14s | %14s" + print(t % ("requested", "returned", "count")) + print(t % (('-' * 14,) * 3)) + + for xy, count in data: + x, y = xy + if x == -2: + x = 'NULL' + elif x == -4: + x = '*' + print(t % (x, y, count)) + + +def print_counts(count_data): + all_attrs = Counter() + for c in count_data: + all_attrs.update(c[1]) + + print("found %d attrs" % len(all_attrs)) + longest = max(len(x) for x in all_attrs) + + #pprint(all_attrs) + rows = OrderedDict() + for a, _ in all_attrs.most_common(): + rows[a] = [a] + + for col_name, counts, doc in count_data: + for attr, row in rows.items(): + d = counts.get(attr, '') + row.append(d) + + print("%15s: %s" % (col_name, doc)) + print() + + t = "%{}s".format(longest) + for c in count_data: + t += " | %{}s".format(max(len(c[0]), 7)) + + h = t % (("attribute",) + tuple(c[0] for c in count_data)) + print(h) + print("-" * len(h)) + + for attr, row in rows.items(): + print(t % tuple(row)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('LDB_PRIVATE_DIR', + help="read attr counts in this directory") + parser.add_argument('--plot', action="store_true", + help='attempt to draw graphs') + parser.add_argument('--no-casefold', action="store_false", + default=True, dest="casefold", + help='See all the encountered case variants') + args = parser.parse_args() + + if not os.path.isdir(args.LDB_PRIVATE_DIR): + parser.print_usage() + sys.exit(1) + + count_data = [] + pair_data = [] + for k, filename, unpacker, doc in DATABASES: + filename = os.path.join(args.LDB_PRIVATE_DIR, filename) + try: + d = unpacker(filename, casefold=args.casefold) + except (RuntimeError, IOError) as e: + print("could not parse %s: %s" % (filename, e)) + continue + if unpacker is unpack_ssize_t_pair: + pair_data.append((k, d, doc)) + else: + count_data.append((k, d, doc)) + + for k, v, doc in pair_data: + if args.plot: + plot_pair_data(k, v, doc) + print_pair_data(k, v, doc) + + print() + print_counts(count_data) + +main() diff --git a/script/autobuild.py b/script/autobuild.py new file mode 100755 index 0000000..ecec352 --- /dev/null +++ b/script/autobuild.py @@ -0,0 +1,1902 @@ +#!/usr/bin/env python3 +# run tests on all Samba subprojects and push to a git tree on success +# Copyright Andrew Tridgell 2010 +# released under GNU GPL v3 or later + +from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError +import os +import tarfile +import sys +import time +import random +from optparse import OptionParser +import smtplib +import email +from email.mime.text import MIMEText +from email.mime.base import MIMEBase +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from sysconfig import get_path +import platform + +import logging + +try: + from waflib.Build import CACHE_SUFFIX +except ImportError: + sys.path.insert(0, "./third_party/waf") + from waflib.Build import CACHE_SUFFIX + +logging.basicConfig(format='%(asctime)s %(message)s') +logger = logging.getLogger('autobuild') +logger.setLevel(logging.INFO) + +os.environ["PYTHONUNBUFFERED"] = "1" + +# This speeds up testing remarkably. +os.environ['TDB_NO_FSYNC'] = '1' + +# allow autobuild to run within git rebase -i +if "GIT_DIR" in os.environ: + del os.environ["GIT_DIR"] +if "GIT_WORK_TREE" in os.environ: + del os.environ["GIT_WORK_TREE"] + +def find_git_root(): + '''get to the top of the git repo''' + p = os.getcwd() + while p != '/': + if os.path.exists(os.path.join(p, ".git")): + return p + p = os.path.abspath(os.path.join(p, '..')) + return None + + +gitroot = find_git_root() +if gitroot is None: + raise Exception("Failed to find git root") + + +def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER')) + +parser = OptionParser() +parser.add_option("--tail", help="show output while running", default=False, action="store_true") +parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true") +parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true") +parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true") +parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase, + default=def_testbase) +parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase, + default=None) +parser.add_option("--passcmd", help="command to run on success", default=None) +parser.add_option("--verbose", help="show all commands as they are run", + default=False, action="store_true") +parser.add_option("--rebase", help="rebase on the given tree before testing", + default=None, type='str') +parser.add_option("--pushto", help="push to a git url on success", + default=None, type='str') +parser.add_option("--mark", help="add a Tested-By signoff before pushing", + default=False, action="store_true") +parser.add_option("--fix-whitespace", help="fix whitespace on rebase", + default=False, action="store_true") +parser.add_option("--retry", help="automatically retry if master changes", + default=False, action="store_true") +parser.add_option("--email", help="send email to the given address on failure", + type='str', default=None) +parser.add_option("--email-from", help="send email from the given address", + type='str', default="autobuild@samba.org") +parser.add_option("--email-server", help="send email via the given server", + type='str', default='localhost') +parser.add_option("--always-email", help="always send email, even on success", + action="store_true") +parser.add_option("--daemon", help="daemonize after initial setup", + action="store_true") +parser.add_option("--branch", help="the branch to work on (default=master)", + default="master", type='str') +parser.add_option("--log-base", help="location where the logs can be found (default=cwd)", + default=gitroot, type='str') +parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?", + default=False, action="store_true") +parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex", + default='') +parser.add_option("--enable-coverage", dest='enable_coverage', + action="store_const", const='--enable-coverage', default='', + help="Add --enable-coverage option while configure") + +(options, args) = parser.parse_args() + +if options.retry: + if options.rebase is None: + raise Exception('You can only use --retry if you also rebase') + +if options.verbose: + logger.setLevel(logging.DEBUG) + +if options.full_testbase is not None: + testbase = options.full_testbase +else: + testbase = "%s/b%u" % (options.testbase, os.getpid()) +test_master = "%s/master" % testbase +test_prefix = "%s/prefix" % testbase +test_tmpdir = "%s/tmp" % testbase +os.environ['TMPDIR'] = test_tmpdir + +if options.enable_coverage: + LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'" +else: + LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"' + +if options.enable_coverage: + PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs" +else: + PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"' + +CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh" + + +def check_symbols(sofile, expected_symbols=""): + return "objdump --dynamic-syms " + sofile + " | " + \ + "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'" + +if args: + # If we are only running specific test, + # do not sleep randomly to wait for it to start + def random_sleep(low, high): + return 'sleep 1' +else: + def random_sleep(low, high): + return 'sleep {}'.format(random.randint(low, high)) + +cleanup_list = [] + +builddirs = { + "ctdb": "ctdb", + "ldb": "lib/ldb", + "tdb": "lib/tdb", + "talloc": "lib/talloc", + "replace": "lib/replace", + "tevent": "lib/tevent", + "pidl": "pidl", + "docs-xml": "docs-xml" +} + +ctdb_configure_params = " --enable-developer ${PREFIX}" +samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data" + +samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH" +samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig" +samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'" +samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}" +samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE" +samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt" +samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + + +def format_option(name, value=None): + """Format option as str list.""" + if value is None: # boolean option + return [name] + if not isinstance(value, list): # single value option + value = [value] + # repeatable option + return ['{}={}'.format(name, item) for item in value] + + +def make_test( + cmd='make testonly', + INJECT_SELFTEST_PREFIX=1, + TESTS='', + include_envs=None, + exclude_envs=None): + + test_options = [] + if include_envs: + test_options = format_option('--include-env', include_envs) + if exclude_envs: + test_options = format_option('--exclude-env', exclude_envs) + if test_options: + # join envs options to original test options + TESTS = (TESTS + ' ' + ' '.join(test_options)).strip() + + _options = [] + + # Allow getting a full CI with + # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0' + + FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1") + + if int(FAIL_IMMEDIATELY): + _options.append('FAIL_IMMEDIATELY=1') + if TESTS: + _options.append("TESTS='{}'".format(TESTS)) + + if INJECT_SELFTEST_PREFIX: + _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}")) + _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}")) + + return ' '.join([cmd] + _options) + + +# When updating this list, also update .gitlab-ci.yml to add the job +# and to make it a dependency of 'page' for the coverage report. + +tasks = { + "ctdb": { + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("configure", "./configure " + ctdb_configure_params), + ("make", "make all"), + ("install", "make install"), + ("test", "make autotest"), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("clean", "make clean"), + ], + }, + "docs-xml": { + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("autoconf", "autoconf"), + ("configure", "./configure"), + ("make", "make html htmlman"), + ("publish-docs", PUBLISH_DOCS), + ("clean", "make clean"), + ], + }, + + "samba-def-build": { + "git-clone-required": True, + "sequence": [ + ("configure", "./configure.developer" + samba_configure_params), + ("make", "make -j"), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("chmod-R-a-w", "chmod -R a-w ."), + ], + }, + + "samba-mit-build": { + "git-clone-required": True, + "sequence": [ + ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params), + ("make", "make -j"), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("chmod-R-a-w", "chmod -R a-w ."), + ], + }, + + "samba-nt4-build": { + "git-clone-required": True, + "sequence": [ + ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params), + ("make", "make -j"), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("chmod-R-a-w", "chmod -R a-w ."), + ], + }, + + "samba-h5l-build": { + "git-clone-required": True, + "sequence": [ + ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params), + ("make", "make -j"), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("chmod-R-a-w", "chmod -R a-w ."), + ], + }, + + "samba-without-smb1-build": { + "git-clone-required": True, + "sequence": [ + ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params), + ("make", "make -j"), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("chmod-R-a-w", "chmod -R a-w ."), + ], + }, + + "samba-no-opath-build": { + "git-clone-required": True, + "sequence": [ + ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1 -DDISABLE_VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS=1 -DDISABLE_PROC_FDS=1' ./configure.developer --without-ad-dc " + samba_configure_params), + ("make", "make -j"), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("chmod-R-a-w", "chmod -R a-w ."), + ], + }, + + # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)' + "samba": { + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("configure", "./configure.developer" + samba_configure_params), + ("make", "make -j"), + ("test", make_test(exclude_envs=[ + "none", + "nt4_dc", + "nt4_dc_smb1", + "nt4_dc_smb1_done", + "nt4_dc_schannel", + "nt4_member", + "ad_dc", + "ad_dc_smb1", + "ad_dc_smb1_done", + "ad_dc_backup", + "ad_dc_ntvfs", + "ad_dc_default", + "ad_dc_default_smb1", + "ad_dc_slowtests", + "ad_dc_no_nss", + "ad_dc_no_ntlm", + "fl2003dc", + "fl2008dc", + "fl2008r2dc", + "ad_member", + "ad_member_idmap_rid", + "admem_idmap_autorid", + "ad_member_idmap_ad", + "ad_member_rfc2307", + "ad_member_idmap_nss", + "ad_member_oneway", + "chgdcpass", + "vampire_2000_dc", + "fl2000dc", + "fileserver", + "fileserver_smb1", + "fileserver_smb1_done", + "maptoguest", + "simpleserver", + "backupfromdc", + "restoredc", + "renamedc", + "offlinebackupdc", + "labdc", + "preforkrestartdc", + "proclimitdc", + "promoted_dc", + "vampire_dc", + "rodc", + "ad_dc_default", + "ad_dc_default_smb1", + "ad_dc_default_smb1_done", + "ad_dc_slowtests", + "schema_pair_dc", + "schema_dc", + "clusteredmember", + "ad_dc_fips", + "ad_member_fips", + ])), + ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])), + ("lcov", LCOV_CMD), + ("install", "make install"), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("clean", "make clean"), + ], + }, + + # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)' + "samba-mitkrb5": { + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params), + ("make", "make -j"), + ("test", make_test(exclude_envs=[ + "none", + "nt4_dc", + "nt4_dc_smb1", + "nt4_dc_smb1_done", + "nt4_dc_schannel", + "nt4_member", + "ad_dc", + "ad_dc_smb1", + "ad_dc_smb1_done", + "ad_dc_backup", + "ad_dc_ntvfs", + "ad_dc_default", + "ad_dc_default_smb1", + "ad_dc_default_smb1_done", + "ad_dc_slowtests", + "ad_dc_no_nss", + "ad_dc_no_ntlm", + "fl2003dc", + "fl2008dc", + "fl2008r2dc", + "ad_member", + "ad_member_idmap_rid", + "admem_idmap_autorid", + "ad_member_idmap_ad", + "ad_member_rfc2307", + "ad_member_idmap_nss", + "ad_member_oneway", + "chgdcpass", + "vampire_2000_dc", + "fl2000dc", + "fileserver", + "fileserver_smb1", + "fileserver_smb1_done", + "maptoguest", + "simpleserver", + "backupfromdc", + "restoredc", + "renamedc", + "offlinebackupdc", + "labdc", + "preforkrestartdc", + "proclimitdc", + "promoted_dc", + "vampire_dc", + "rodc", + "ad_dc_default", + "ad_dc_default_smb1", + "ad_dc_default_smb1_done", + "ad_dc_slowtests", + "schema_pair_dc", + "schema_dc", + "clusteredmember", + "ad_dc_fips", + "ad_member_fips", + ])), + ("lcov", LCOV_CMD), + ("install", "make install"), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("clean", "make clean"), + ], + }, + + "samba-nt4": { + "dependency": "samba-nt4-build", + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("test", make_test(include_envs=[ + "nt4_dc", + "nt4_dc_smb1", + "nt4_dc_smb1_done", + "nt4_dc_schannel", + "nt4_member", + "simpleserver", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-fileserver": { + "dependency": "samba-h5l-build", + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("test", make_test(include_envs=[ + "fileserver", + "fileserver_smb1", + "fileserver_smb1_done", + "maptoguest", + "ktest", # ktest is also tested in samba-ktest-mit samba + # and samba-mitkrb5 but is tested here against + # a system Heimdal + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-fileserver-without-smb1": { + "dependency": "samba-without-smb1-build", + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("test", make_test(include_envs=["fileserver"])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + # This is a full build without the AD DC so we test the build with + # MIT Kerberos from the current system. Runtime behaviour is + # confirmed via the ktest (static ccache and keytab) environment + + # This environment also used to confirm we can still build with --with-libunwind + "samba-ktest-mit": { + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params), + ("make", "make -j"), + ("test", make_test(include_envs=[ + "ktest", # ktest is also tested in fileserver, samba and + # samba-mitkrb5 but is tested here against a + # system MIT krb5 + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-admem": { + "dependency": "samba-def-build", + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("test", make_test(include_envs=[ + "ad_member", + "ad_member_idmap_rid", + "admem_idmap_autorid", + "ad_member_idmap_ad", + "ad_member_rfc2307", + "ad_member_idmap_nss", + "ad_member_offlogon", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-no-opath1": { + "dependency": "samba-no-opath-build", + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("test", make_test( + cmd="make testonly DISABLE_OPATH=1", + include_envs=[ + "nt4_dc", + "nt4_dc_smb1", + "nt4_dc_smb1_done", + "nt4_dc_schannel", + "nt4_member", + "simpleserver", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", "script/clean-source-tree.sh"), + ], + }, + + "samba-no-opath2": { + "dependency": "samba-no-opath-build", + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("test", make_test( + cmd="make testonly DISABLE_OPATH=1", + include_envs=[ + "fileserver", + "fileserver_smb1", + "fileserver_smb1_done", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", "script/clean-source-tree.sh"), + ], + }, + + "samba-ad-dc-1": { + "dependency": "samba-def-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=[ + "ad_dc", + "ad_dc_smb1", + "ad_dc_smb1_done", + "ad_dc_no_nss", + "ad_dc_no_ntlm", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-ad-dc-2": { + "dependency": "samba-def-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=[ + "vampire_dc", + "vampire_2000_dc", + "rodc", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-ad-dc-3": { + "dependency": "samba-def-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=[ + "promoted_dc", + "chgdcpass", + "preforkrestartdc", + "proclimitdc", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-ad-dc-4a": { + "dependency": "samba-def-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=[ + "fl2000dc", + "ad_member_oneway", + "fl2003dc", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + "samba-ad-dc-4b": { + "dependency": "samba-def-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=[ + "fl2008dc", + "fl2008r2dc", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-ad-dc-5": { + "dependency": "samba-def-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=[ + "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-ad-dc-6": { + "dependency": "samba-def-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-schemaupgrade": { + "dependency": "samba-def-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait + # This is currently the longest task, so we don't randomly delay it. + "samba-ad-dc-ntvfs": { + "dependency": "samba-def-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=["ad_dc_ntvfs"])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + # Test fips compliance + "samba-fips": { + "dependency": "samba-mit-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])), + # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + # run the backup/restore testenvs separately as they're fairly standalone + # (and CI seems to max out at ~3 different DCs running at once) + "samba-ad-back1": { + "dependency": "samba-def-build", + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("test", make_test(include_envs=[ + "backupfromdc", + "restoredc", + "renamedc", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + "samba-ad-back2": { + "dependency": "samba-def-build", + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("test", make_test(include_envs=[ + "backupfromdc", + "offlinebackupdc", + "labdc", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-admem-mit": { + "dependency": "samba-mit-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=[ + "ad_member", + "ad_member_idmap_rid", + "admem_idmap_autorid", + "ad_member_idmap_ad", + "ad_member_rfc2307", + "ad_member_idmap_nss", + "ad_member_offlogon", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-addc-mit-1": { + "dependency": "samba-mit-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=[ + "ad_dc", + "ad_dc_smb1", + "ad_dc_smb1_done", + "ad_dc_no_nss", + "ad_dc_no_ntlm", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-addc-mit-4a": { + "dependency": "samba-mit-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=[ + "fl2000dc", + "ad_member_oneway", + "fl2003dc", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + "samba-addc-mit-4b": { + "dependency": "samba-mit-build", + "sequence": [ + ("random-sleep", random_sleep(1, 1)), + ("test", make_test(include_envs=[ + "fl2008dc", + "fl2008r2dc", + ])), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ], + }, + + "samba-test-only": { + "sequence": [ + ("configure", "./configure.developer --abi-check-disable" + samba_configure_params), + ("make", "make -j"), + ("test", make_test(TESTS="${TESTS}")), + ("lcov", LCOV_CMD), + ], + }, + + # Test cross-compile infrastructure + "samba-xc": { + "sequence": [ + ("random-sleep", random_sleep(900, 1500)), + ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), + ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \ + " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params), + ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"), + ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \ + " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params), + ("compare-results", "script/compare_cc_results.py " + "./bin/c4che/default{} " + "./bin-xe/c4che/default{} " + "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))), + ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"), + ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \ + " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params), + ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \ + " = \"'1234'\"".format(CACHE_SUFFIX)), + ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"), + ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \ + " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \ + " ; test $? -ne 0"), + ], + }, + + # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments + "samba-o3": { + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params), + ("make", "make -j"), + ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])), + ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])), + ("lcov", LCOV_CMD), + ("install", "make install"), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("clean", "make clean"), + ], + }, + + "samba-32bit": { + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("configure", "./configure.developer --abi-check-disable --disable-warnings-as-errors" + samba_configure_params), + ("make", "make -j"), + ("nonetest", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])), + ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])), + ("ktest", make_test(cmd='make test', include_envs=["ktest"])), + ("install", "make install"), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("clean", "make clean"), + ], + }, + + "samba-ctdb": { + "sequence": [ + ("random-sleep", random_sleep(900, 1500)), + + # make sure we have tdb around: + ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"), + ("tdb-make", "cd lib/tdb && make"), + ("tdb-install", "cd lib/tdb && make install"), + + # build samba with cluster support (also building ctdb): + ("samba-configure", + "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH " + "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} " + "./configure.developer ${PREFIX} " + "--with-selftest-prefix=./bin/ab " + "--with-cluster-support " + "--without-ad-dc " + "--bundled-libraries=!tdb"), + ("samba-make", "make"), + ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"), + ("samba-install", "make install"), + ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"), + + ("test", make_test( + cmd='PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH make test', + INJECT_SELFTEST_PREFIX=0, + include_envs=["clusteredmember"]) + ), + + # clean up: + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("clean", "make clean"), + ("ctdb-clean", "cd ./ctdb && make clean"), + ], + }, + + "samba-libs": { + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs), + ("talloc-make", "cd lib/talloc && make"), + ("talloc-install", "cd lib/talloc && make install"), + + ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs), + ("tdb-make", "cd lib/tdb && make"), + ("tdb-install", "cd lib/tdb && make install"), + + ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs), + ("tevent-make", "cd lib/tevent && make"), + ("tevent-install", "cd lib/tevent && make install"), + + ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs), + ("ldb-make", "cd lib/ldb && make"), + ("ldb-install", "cd lib/ldb && make install"), + + ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"), + ("nondevel-make", "make -j"), + ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"), + ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"), + ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"), + ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"), + ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"), + ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"), + ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"), + ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"), + ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"), + ("nondevel-no-public-nss_winbind", + check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")), + ("nondevel-no-public-nss_wins", + check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")), + ("nondevel-no-public-libwbclient", + check_symbols("./bin/shared/libwbclient.so.0", "wbc")), + ("nondevel-no-public-pam_winbind", + check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")), + ("nondevel-no-public-winbind_krb5_locator", + check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")), + ("nondevel-no-public-async_dns_krb5_locator", + check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")), + ("nondevel-install", "make -j install"), + ("nondevel-dist", "make dist"), + + ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"), + ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"), + ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"), + ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"), + ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"), + ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"), + ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"), + ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"), + ("prefix-no-public-nss_winbind", + check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")), + ("prefix-no-public-nss_wins", + check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")), + ("prefix-no-public-libwbclient", + check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")), + ("prefix-no-public-pam_winbind", + check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")), + ("prefix-no-public-winbind_krb5_locator", + check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so", + "service_locator")), + ("prefix-no-public-async_dns_krb5_locator", + check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so", + "service_locator")), + + # retry with all modules shared + ("allshared-distclean", "make distclean"), + ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"), + ("allshared-make", "make -j"), + ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"), + ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"), + ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"), + ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"), + ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"), + ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"), + ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"), + ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"), + ("allshared-no-public-nss_winbind", + check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")), + ("allshared-no-public-nss_wins", + check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")), + ("allshared-no-public-libwbclient", + check_symbols("./bin/shared/libwbclient.so.0", "wbc")), + ("allshared-no-public-pam_winbind", + check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")), + ("allshared-no-public-winbind_krb5_locator", + check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")), + ("allshared-no-public-async_dns_krb5_locator", + check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")), + ], + }, + + "samba-fuzz": { + "sequence": [ + # build the fuzzers (static) via the oss-fuzz script + ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"), + ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"), + ], + }, + + # * Test smbd and smbtorture can build semi-static + # + # * Test Samba without python still builds. + # + # When this test fails due to more use of Python, the expectations + # is that the newly failing part of the code should be disabled + # when --disable-python is set (rather than major work being done + # to support this environment). + # + # The target here is for vendors shipping a minimal smbd. + "samba-minimal-smbd": { + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + + # build with all modules static + ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"), + ("allstatic-make", "make -j"), + ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")), + ("allstatic-lcov", LCOV_CMD), + ("allstatic-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("allstatic-def-clean", "make clean"), + + # force all libraries as private + ("allprivate-def-distclean", "make distclean"), + ("allprivate-def-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL"), + ("allprivate-def-make", "make -j"), + # note wrapper libraries need to be public + ("allprivate-def-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so' | wc -l | grep -q '^0'"), + ("allprivate-def-only-private-ext", "ls ./bin/shared/private | egrep 'private-samba' | wc -l | grep -q '^0' && exit 1; exit 0"), + ("allprivate-def-no-non-private-ext", "ls ./bin/shared/private | egrep -v 'private-samba|^libpypamtest.so$' | wc -l | grep -q '^0'"), + ("allprivate-def-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")), + ("allprivate-def-lcov", LCOV_CMD), + ("allprivate-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("allprivate-def-clean", "make clean"), + + # force all libraries as private with a non default + # extension and 2 exceptions + ("allprivate-ext-distclean", "make distclean"), + ("allprivate-ext-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL --private-library-extension=private-library --private-extension-exception=pac,ndr"), + ("allprivate-ext-make", "make -j"), + # note wrapper libraries need to be public + ("allprivate-ext-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so' | wc -l | grep -q '^0'"), + ("allprivate-ext-no-private-default-ext", "ls ./bin/shared/private | grep 'private-samba' | wc -l | grep -q '^0'"), + ("allprivate-ext-has-private-ext", "ls ./bin/shared/private | grep 'private-library' | wc -l | grep -q '^0' && exit 1; exit 0"), + ("allprivate-ext-libndr-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libndr' | wc -l | grep -q '^1'"), + ("allprivate-ext-libpac-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libpac' | wc -l | grep -q '^1'"), + ("allprivate-ext-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")), + ("allprivate-ext-lcov", LCOV_CMD), + ("allprivate-ext-check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("allprivate-ext-clean", "make clean"), + + # retry with nonshared smbd and smbtorture + ("nonshared-distclean", "make distclean"), + ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"), + ("nonshared-make", "make -j"), + ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")), + ("nonshared-lcov", LCOV_CMD), + ("nonshared-check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("nonshared-clean", "make clean"), + ], + }, + + "samba-nopython": { + "sequence": [ + ("random-sleep", random_sleep(300, 900)), + + ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"), + ("make", "make -j"), + ("find-python", "script/find_python.sh ${PREFIX}"), + ("test", "make test-nopython"), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("clean", "make clean"), + + ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"), + ("talloc-make", "cd lib/talloc && make"), + ("talloc-install", "cd lib/talloc && make install"), + + ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"), + ("tdb-make", "cd lib/tdb && make"), + ("tdb-install", "cd lib/tdb && make install"), + + ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"), + ("tevent-make", "cd lib/tevent && make"), + ("tevent-install", "cd lib/tevent && make install"), + + ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"), + ("ldb-make", "cd lib/ldb && make"), + ("ldb-install", "cd lib/ldb && make install"), + + # retry against installed library packages, but no required modules + ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"), + ("libs-make", "make -j"), + ("libs-install", "make install"), + ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("libs-clean", "make clean"), + + ], + }, + + "samba-codecheck": { + "sequence": [ + ("run", "script/check-shell-scripts.sh ."), + ("run", "script/codespell.sh ."), + ], + }, + + "ldb": { + "sequence": [ + ("random-sleep", random_sleep(60, 600)), + ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"), + ("make", "make"), + ("install", "make install"), + ("test", "make test"), + ("lcov", LCOV_CMD), + ("clean", "make clean"), + ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"), + ("make-no-lmdb", "make"), + ("test-no-lmdb", "make test"), + ("lcov-no-lmdb", LCOV_CMD), + ("install-no-lmdb", "make install"), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("distcheck", "make distcheck"), + ("clean", "make clean"), + ], + }, + + "tdb": { + "sequence": [ + ("random-sleep", random_sleep(60, 600)), + ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"), + ("make", "make"), + ("install", "make install"), + ("test", "make test"), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("distcheck", "make distcheck"), + ("clean", "make clean"), + ], + }, + + "talloc": { + "sequence": [ + ("random-sleep", random_sleep(60, 600)), + ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"), + ("make", "make"), + ("install", "make install"), + ("test", "make test"), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("distcheck", "make distcheck"), + ("clean", "make clean"), + ], + }, + + "replace": { + "sequence": [ + ("random-sleep", random_sleep(60, 600)), + ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"), + ("make", "make"), + ("install", "make install"), + ("test", "make test"), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("distcheck", "make distcheck"), + ("clean", "make clean"), + ], + }, + + "tevent": { + "sequence": [ + ("random-sleep", random_sleep(60, 600)), + ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"), + ("make", "make"), + ("install", "make install"), + ("test", "make test"), + ("lcov", LCOV_CMD), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("distcheck", "make distcheck"), + ("clean", "make clean"), + ], + }, + + "pidl": { + "git-clone-required": True, + "sequence": [ + ("random-sleep", random_sleep(60, 600)), + ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"), + ("touch", "touch *.yp"), + ("make", "make"), + ("test", "make test"), + ("install", "make install"), + ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"), + ("check-clean-tree", CLEAN_SOURCE_TREE_CMD), + ("clean", "make clean"), + ], + }, + + # these are useful for debugging autobuild + "pass": { + "sequence": [ + ("pass", 'echo passing && /bin/true'), + ], + }, + "fail": { + "sequence": [ + ("fail", 'echo failing && /bin/false'), + ], + }, +} + +defaulttasks = list(tasks.keys()) + +defaulttasks.remove("pass") +defaulttasks.remove("fail") + +# The build tasks will be brought in by the test tasks as needed +defaulttasks.remove("samba-def-build") +defaulttasks.remove("samba-nt4-build") +defaulttasks.remove("samba-mit-build") +defaulttasks.remove("samba-h5l-build") +defaulttasks.remove("samba-no-opath-build") + +# This is not a normal test, but a task to support manually running +# one test under autobuild +defaulttasks.remove("samba-test-only") + +# Only built on GitLab CI and not in the default autobuild because it +# uses too much space (4GB of semi-static binaries) +defaulttasks.remove("samba-fuzz") + +# The FIPS build runs only in GitLab CI on a current Fedora Docker +# container where a simulated FIPS mode is possible. +defaulttasks.remove("samba-fips") + +# The MIT build runs on a current Fedora where an up to date MIT KDC +# is already packaged. This avoids needing to backport a current MIT +# to the default Ubuntu 18.04, particularly during development, and +# the need to install on the shared sn-devel-184. + +defaulttasks.remove("samba-mitkrb5") +defaulttasks.remove("samba-admem-mit") +defaulttasks.remove("samba-addc-mit-1") +defaulttasks.remove("samba-addc-mit-4a") +defaulttasks.remove("samba-addc-mit-4b") + +defaulttasks.remove("samba-32bit") + +if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1": + defaulttasks.remove("samba-o3") + + +def do_print(msg): + logger.info(msg) + sys.stdout.flush() + sys.stderr.flush() + +def do_debug(msg): + logger.debug(msg) + sys.stdout.flush() + sys.stderr.flush() + + +def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True): + if show is None: + do_debug("Running: '%s' in '%s'" % (cmd, dir)) + elif show: + do_print("Running: '%s' in '%s'" % (cmd, dir)) + + if output: + out = check_output([cmd], shell=True, cwd=dir) + return out.decode(encoding='utf-8', errors='backslashreplace') + elif checkfail: + return check_call(cmd, shell=True, cwd=dir) + else: + return call(cmd, shell=True, cwd=dir) + +def rmdir_force(dirname, re_raise=True): + try: + run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % ( + dirname, dirname, dirname), output=True, show=True) + except CalledProcessError as e: + do_print("Failed: '%s'" % (str(e))) + run_cmd("tree %s" % dirname, output=True, show=True) + if re_raise: + raise + return False + return True + +class builder(object): + '''handle build of one directory''' + + def __init__(self, name, definition): + self.name = name + self.dir = builddirs.get(name, '.') + self.tag = self.name.replace('/', '_') + self.definition = definition + self.sequence = definition["sequence"] + self.git_clone_required = False + if "git-clone-required" in definition: + self.git_clone_required = bool(definition["git-clone-required"]) + self.proc = None + self.done = False + self.next = 0 + self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag) + self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag) + do_debug("stdout for %s in %s" % (self.name, self.stdout_path)) + do_debug("stderr for %s in %s" % (self.name, self.stderr_path)) + run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path)) + self.stdout = open(self.stdout_path, 'w') + self.stderr = open(self.stderr_path, 'w') + self.stdin = open("/dev/null", 'r') + self.builder_dir = "%s/%s" % (testbase, self.tag) + self.test_source_dir = self.builder_dir + self.cwd = "%s/%s" % (self.builder_dir, self.dir) + self.selftest_prefix = "%s/bin/ab" % (self.cwd) + self.prefix = "%s/%s" % (test_prefix, self.tag) + self.consumers = [] + self.producer = None + + if self.git_clone_required: + assert "dependency" not in definition + + def mark_existing(self): + do_debug('%s: Mark as existing dependency' % self.name) + self.next = len(self.sequence) + self.done = True + + def add_consumer(self, consumer): + do_debug("%s: add consumer: %s" % (self.name, consumer.name)) + consumer.producer = self + consumer.test_source_dir = self.test_source_dir + self.consumers.append(consumer) + + def start_next(self): + if self.producer is not None: + if not self.producer.done: + do_debug("%s: Waiting for producer: %s" % (self.name, self.producer.name)) + return + + if self.next == 0: + rmdir_force(self.builder_dir) + rmdir_force(self.prefix) + if self.producer is not None: + run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True) + elif not self.git_clone_required: + run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True) + else: + run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True) + + if self.next == len(self.sequence): + if not self.done: + do_print('%s: Completed OK' % self.name) + self.done = True + if not options.nocleanup and len(self.consumers) == 0: + do_print('%s: Cleaning up' % self.name) + rmdir_force(self.builder_dir) + rmdir_force(self.prefix) + for consumer in self.consumers: + if consumer.next != 0: + continue + do_print('%s: Starting consumer %s' % (self.name, consumer.name)) + consumer.start_next() + if self.producer is not None: + self.producer.consumers.remove(self) + assert self.producer.done + self.producer.start_next() + do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers))) + return + (self.stage, self.cmd) = self.sequence[self.next] + self.cmd = self.cmd.replace("${PYTHON_PREFIX}", + get_path(name='platlib', + scheme="posix_prefix", + vars={"base": self.prefix, + "platbase": self.prefix})) + self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix) + self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix) + self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests) + self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir) + self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix) + self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base) + self.cmd = self.cmd.replace("${NAME}", self.name) + self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage) + do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd)) + self.proc = Popen(self.cmd, shell=True, + close_fds=True, cwd=self.cwd, + stdout=self.stdout, stderr=self.stderr, stdin=self.stdin) + self.next += 1 + +def expand_dependencies(n): + deps = list() + if "dependency" in tasks[n]: + depname = tasks[n]["dependency"] + assert depname in tasks + sdeps = expand_dependencies(depname) + assert n not in sdeps + for sdep in sdeps: + deps.append(sdep) + deps.append(depname) + return deps + + +class buildlist(object): + '''handle build of multiple directories''' + + def __init__(self, tasknames, rebase_url, rebase_branch="master"): + self.tail_proc = None + self.retry = None + if not tasknames: + if options.restrict_tests: + tasknames = ["samba-test-only"] + else: + tasknames = defaulttasks + + given_tasknames = tasknames.copy() + implicit_tasknames = [] + for n in given_tasknames: + deps = expand_dependencies(n) + for dep in deps: + if dep in given_tasknames: + continue + if dep in implicit_tasknames: + continue + implicit_tasknames.append(dep) + + tasknames = implicit_tasknames.copy() + tasknames.extend(given_tasknames) + do_debug("given_tasknames: %s" % given_tasknames) + do_debug("implicit_tasknames: %s" % implicit_tasknames) + do_debug("tasknames: %s" % tasknames) + self.tlist = [builder(n, tasks[n]) for n in tasknames] + + if options.retry: + rebase_remote = "rebaseon" + retry_task = { + "git-clone-required": True, + "sequence": [ + ("retry", + '''set -e + git remote add -t %s %s %s + git fetch %s + while :; do + sleep 60 + git describe %s/%s > old_remote_branch.desc + git fetch %s + git describe %s/%s > remote_branch.desc + diff old_remote_branch.desc remote_branch.desc + done + ''' % ( + rebase_branch, rebase_remote, rebase_url, + rebase_remote, + rebase_remote, rebase_branch, + rebase_remote, + rebase_remote, rebase_branch + ))]} + + self.retry = builder('retry', retry_task) + self.need_retry = False + + if options.skip_dependencies: + for b in self.tlist: + if b.name in implicit_tasknames: + b.mark_existing() + + for b in self.tlist: + do_debug("b.name=%s" % b.name) + if "dependency" not in b.definition: + continue + depname = b.definition["dependency"] + do_debug("b.name=%s: dependency:%s" % (b.name, depname)) + for p in self.tlist: + if p.name == depname: + p.add_consumer(b) + + def kill_kids(self): + if self.tail_proc is not None: + self.tail_proc.terminate() + self.tail_proc.wait() + self.tail_proc = None + if self.retry is not None: + self.retry.proc.terminate() + self.retry.proc.wait() + self.retry = None + for b in self.tlist: + if b.proc is not None: + run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False) + b.proc.terminate() + b.proc.wait() + b.proc = None + + def wait_one(self): + while True: + none_running = True + for b in self.tlist: + if b.proc is None: + continue + none_running = False + b.status = b.proc.poll() + if b.status is None: + continue + b.proc = None + return b + if options.retry: + ret = self.retry.proc.poll() + if ret is not None: + self.need_retry = True + self.retry = None + return None + if none_running: + return None + time.sleep(0.1) + + def run(self): + for b in self.tlist: + b.start_next() + if options.retry: + self.retry.start_next() + while True: + b = self.wait_one() + if options.retry and self.need_retry: + self.kill_kids() + do_print("retry needed") + return (0, None, None, None, "retry") + if b is None: + break + if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0: + self.kill_kids() + return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status)) + b.start_next() + self.kill_kids() + return (0, None, None, None, "All OK") + + def write_system_info(self, filename): + with open(filename, 'w') as f: + for cmd in ['uname -a', + 'lsb_release -a', + 'free', + 'mount', + 'cat /proc/cpuinfo', + 'cc --version', + 'df -m .', + 'df -m %s' % testbase]: + try: + out = run_cmd(cmd, output=True, checkfail=False) + except CalledProcessError as e: + out = "<failed: %s>" % str(e) + print('### %s' % cmd, file=f) + print(out, file=f) + print(file=f) + + def tarlogs(self, fname): + with tarfile.open(fname, "w:gz") as tar: + for b in self.tlist: + tar.add(b.stdout_path, arcname="%s.stdout" % b.tag) + tar.add(b.stderr_path, arcname="%s.stderr" % b.tag) + if os.path.exists("autobuild.log"): + tar.add("autobuild.log") + filename = 'system-info.txt' + self.write_system_info(filename) + tar.add(filename) + + def remove_logs(self): + for b in self.tlist: + os.unlink(b.stdout_path) + os.unlink(b.stderr_path) + + def start_tail(self): + cmd = ["tail", "-f"] + for b in self.tlist: + cmd.append(b.stdout_path) + cmd.append(b.stderr_path) + self.tail_proc = Popen(cmd, close_fds=True) + + +def cleanup(do_raise=False): + if options.nocleanup: + return + run_cmd("stat %s || true" % test_tmpdir, show=True) + run_cmd("stat %s" % testbase, show=True) + do_print("Cleaning up %r" % cleanup_list) + for d in cleanup_list: + ok = rmdir_force(d, re_raise=False) + if ok: + continue + if os.path.isdir(d): + do_print("Killing, waiting and retry") + run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False) + else: + do_print("Waiting and retry") + time.sleep(1) + rmdir_force(d, re_raise=do_raise) + + +def daemonize(logfile): + pid = os.fork() + if pid == 0: # Parent + os.setsid() + pid = os.fork() + if pid != 0: # Actual daemon + os._exit(0) + else: # Grandparent + os._exit(0) + + import resource # Resource usage information. + maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] + if maxfd == resource.RLIM_INFINITY: + maxfd = 1024 # Rough guess at maximum number of open file descriptors. + for fd in range(0, maxfd): + try: + os.close(fd) + except OSError: + pass + os.open(logfile, os.O_RDWR | os.O_CREAT) + os.dup2(0, 1) + os.dup2(0, 2) + + +def write_pidfile(fname): + '''write a pid file, cleanup on exit''' + with open(fname, mode='w') as f: + f.write("%u\n" % os.getpid()) + + +def rebase_tree(rebase_url, rebase_branch="master"): + rebase_remote = "rebaseon" + do_print("Rebasing on %s" % rebase_url) + run_cmd("git describe HEAD", show=True, dir=test_master) + run_cmd("git remote add -t %s %s %s" % + (rebase_branch, rebase_remote, rebase_url), + show=True, dir=test_master) + run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master) + if options.fix_whitespace: + run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" % + (rebase_remote, rebase_branch), + show=True, dir=test_master) + else: + run_cmd("git rebase --force-rebase %s/%s" % + (rebase_remote, rebase_branch), + show=True, dir=test_master) + diff = run_cmd("git --no-pager diff HEAD %s/%s" % + (rebase_remote, rebase_branch), + dir=test_master, output=True) + if diff == '': + do_print("No differences between HEAD and %s/%s - exiting" % + (rebase_remote, rebase_branch)) + sys.exit(0) + run_cmd("git describe %s/%s" % + (rebase_remote, rebase_branch), + show=True, dir=test_master) + run_cmd("git describe HEAD", show=True, dir=test_master) + run_cmd("git --no-pager diff --stat HEAD %s/%s" % + (rebase_remote, rebase_branch), + show=True, dir=test_master) + + +def push_to(push_url, push_branch="master"): + push_remote = "pushto" + do_print("Pushing to %s" % push_url) + if options.mark: + run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master) + run_cmd("git commit --amend -c HEAD", dir=test_master) + # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master + # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master) + run_cmd("git remote add -t %s %s %s" % + (push_branch, push_remote, push_url), + show=True, dir=test_master) + run_cmd("git push %s +HEAD:%s" % + (push_remote, push_branch), + show=True, dir=test_master) + + +def send_email(subject, text, log_tar): + if options.email is None: + do_print("not sending email because the recipient is not set") + do_print("the text content would have been:\n\nSubject: %s\n\n%s" % + (subject, text)) + return + outer = MIMEMultipart() + outer['Subject'] = subject + outer['To'] = options.email + outer['From'] = options.email_from + outer['Date'] = email.utils.formatdate(localtime=True) + outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n' + outer.attach(MIMEText(text, 'plain', 'utf-8')) + if options.attach_logs: + with open(log_tar, 'rb') as fp: + msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64) + # Set the filename parameter + msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar)) + outer.attach(msg) + content = outer.as_string() + s = smtplib.SMTP(options.email_server) + email_user = os.getenv('SMTP_USERNAME') + email_password = os.getenv('SMTP_PASSWORD') + if email_user is not None: + s.starttls() + s.login(email_user, email_password) + + s.sendmail(options.email_from, [options.email], content) + s.set_debuglevel(1) + s.quit() + + +def email_failure(status, failed_task, failed_stage, failed_tag, errstr, + elapsed_time, log_base=None, add_log_tail=True): + '''send an email to options.email about the failure''' + elapsed_minutes = elapsed_time / 60.0 + if log_base is None: + log_base = gitroot + text = ''' +Dear Developer, + +Your autobuild on %s failed after %.1f minutes +when trying to test %s with the following error: + + %s + +the autobuild has been abandoned. Please fix the error and resubmit. + +A summary of the autobuild process is here: + + %s/autobuild.log +''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base) + + if options.restrict_tests: + text += """ +The build was restricted to tests matching %s\n""" % options.restrict_tests + + if failed_task != 'rebase': + text += ''' +You can see logs of the failed task here: + + %s/%s.stdout + %s/%s.stderr + +or you can get full logs of all tasks in this job here: + + %s/logs.tar.gz + +The top commit for the tree that was built was: + +%s + +''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg) + + log_stdout = "%s/%s.stdout" % (gitroot, failed_tag) + if add_log_tail and os.access(log_stdout, os.R_OK): + f = open(log_stdout, 'r') + lines = f.readlines() + log_tail = "".join(lines[-50:]) + num_lines = len(lines) + log_stderr = "%s/%s.stderr" % (gitroot, failed_tag) + if num_lines < 50 and os.access(log_stderr, os.R_OK): + # Also include stderr (compile failures) if < 50 lines of stdout + f = open(log_stderr, 'r') + log_tail += "".join(f.readlines()[-(50 - num_lines):]) + + text += ''' +The last 50 lines of log messages: + +%s + ''' % log_tail + f.close() + + logs = os.path.join(gitroot, 'logs.tar.gz') + send_email('autobuild[%s] failure on %s for task %s during %s' + % (options.branch, platform.node(), failed_task, failed_stage), + text, logs) + + +def email_success(elapsed_time, log_base=None): + '''send an email to options.email about a successful build''' + if log_base is None: + log_base = gitroot + text = ''' +Dear Developer, + +Your autobuild on %s has succeeded after %.1f minutes. + +''' % (platform.node(), elapsed_time / 60.) + + if options.restrict_tests: + text += """ +The build was restricted to tests matching %s\n""" % options.restrict_tests + + if options.keeplogs: + text += ''' + +you can get full logs of all tasks in this job here: + + %s/logs.tar.gz + +''' % log_base + + text += ''' +The top commit for the tree that was built was: + +%s +''' % top_commit_msg + + logs = os.path.join(gitroot, 'logs.tar.gz') + send_email('autobuild[%s] success on %s' % (options.branch, platform.node()), + text, logs) + + +# get the top commit message, for emails +top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True) + +try: + if options.skip_dependencies: + run_cmd("stat %s" % testbase, dir=testbase, output=True) + else: + os.makedirs(testbase) +except Exception as reason: + raise Exception("Unable to create %s : %s" % (testbase, reason)) +cleanup_list.append(testbase) + +if options.daemon: + logfile = os.path.join(testbase, "log") + do_print("Forking into the background, writing progress to %s" % logfile) + daemonize(logfile) + +write_pidfile(gitroot + "/autobuild.pid") + +start_time = time.time() + +while True: + try: + run_cmd("rm -rf %s" % test_tmpdir, show=True) + os.makedirs(test_tmpdir) + # The waf uninstall code removes empty directories all the way + # up the tree. Creating a file in test_tmpdir stops it from + # being removed. + run_cmd("touch %s" % os.path.join(test_tmpdir, + ".directory-is-not-empty"), show=True) + run_cmd("stat %s" % test_tmpdir, show=True) + run_cmd("stat %s" % testbase, show=True) + if options.skip_dependencies: + run_cmd("stat %s" % test_master, dir=testbase, output=True) + else: + run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot) + except Exception: + cleanup() + raise + + try: + if options.rebase is not None: + rebase_tree(options.rebase, rebase_branch=options.branch) + except Exception: + cleanup_list.append(gitroot + "/autobuild.pid") + cleanup() + elapsed_time = time.time() - start_time + email_failure(-1, 'rebase', 'rebase', 'rebase', + 'rebase on %s failed' % options.branch, + elapsed_time, log_base=options.log_base) + sys.exit(1) + + try: + blist = buildlist(args, options.rebase, rebase_branch=options.branch) + if options.tail: + blist.start_tail() + (status, failed_task, failed_stage, failed_tag, errstr) = blist.run() + if status != 0 or errstr != "retry": + break + cleanup(do_raise=True) + except Exception: + cleanup() + raise + +cleanup_list.append(gitroot + "/autobuild.pid") + +do_print(errstr) + +blist.kill_kids() +if options.tail: + do_print("waiting for tail to flush") + time.sleep(1) + +elapsed_time = time.time() - start_time +if status == 0: + if options.passcmd is not None: + do_print("Running passcmd: %s" % options.passcmd) + run_cmd(options.passcmd, dir=test_master) + if options.pushto is not None: + push_to(options.pushto, push_branch=options.branch) + if options.keeplogs or options.attach_logs: + blist.tarlogs("logs.tar.gz") + do_print("Logs in logs.tar.gz") + if options.always_email: + email_success(elapsed_time, log_base=options.log_base) + blist.remove_logs() + cleanup() + do_print(errstr) + sys.exit(0) + +# something failed, gather a tar of the logs +blist.tarlogs("logs.tar.gz") + +if options.email is not None: + email_failure(status, failed_task, failed_stage, failed_tag, errstr, + elapsed_time, log_base=options.log_base) +else: + elapsed_minutes = elapsed_time / 60.0 + print(''' + +#################################################################### + +AUTOBUILD FAILURE + +Your autobuild[%s] on %s failed after %.1f minutes +when trying to test %s with the following error: + + %s + +the autobuild has been abandoned. Please fix the error and resubmit. + +#################################################################### + +''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr)) + +cleanup() +do_print(errstr) +do_print("Logs in logs.tar.gz") +sys.exit(status) diff --git a/script/bisect-test.py b/script/bisect-test.py new file mode 100755 index 0000000..7c5cd63 --- /dev/null +++ b/script/bisect-test.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# use git bisect to work out what commit caused a test failure +# Copyright Andrew Tridgell 2010 +# released under GNU GPL v3 or later + + +from subprocess import call, check_call, Popen, PIPE +import os +import tempfile +import sys +from optparse import OptionParser + +parser = OptionParser() +parser.add_option("", "--good", help="known good revision (default HEAD~100)", default='HEAD~100') +parser.add_option("", "--bad", help="known bad revision (default HEAD)", default='HEAD') +parser.add_option("", "--skip-build-errors", help="skip revision where make fails", + action='store_true', default=False) +parser.add_option("", "--autogen", help="run autogen before each build", action="store_true", default=False) +parser.add_option("", "--autogen-command", help="command to use for autogen (default ./autogen.sh)", + type='str', default="./autogen.sh") +parser.add_option("", "--configure", help="run configure.developer before each build", + action="store_true", default=False) +parser.add_option("", "--configure-command", help="the command for configure (default ./configure.developer)", + type='str', default="./configure.developer") +parser.add_option("", "--build-command", help="the command to build the tree (default 'make -j')", + type='str', default="make -j") +parser.add_option("", "--test-command", help="the command to test the tree (default 'make test')", + type='str', default="make test") +parser.add_option("", "--clean", help="run make clean before each build", + action="store_true", default=False) + + +(opts, args) = parser.parse_args() + + +def run_cmd(cmd, dir=".", show=True, output=False, checkfail=True): + if show: + print("Running: '%s' in '%s'" % (cmd, dir)) + if output: + return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0] + elif checkfail: + return check_call(cmd, shell=True, cwd=dir) + else: + return call(cmd, shell=True, cwd=dir) + + +def find_git_root(): + '''get to the top of the git repo''' + p = os.getcwd() + while p != '/': + if os.path.exists(os.path.join(p, ".git")): + return p + p = os.path.abspath(os.path.join(p, '..')) + return None + + +cwd = os.getcwd() +gitroot = find_git_root() + +# create a bisect script +f = tempfile.NamedTemporaryFile(delete=False, mode="w+t") +f.write("set -x\n") +f.write("cd %s || exit 125\n" % cwd) +if opts.autogen: + f.write("%s || exit 125\n" % opts.autogen_command) +if opts.configure: + f.write("%s || exit 125\n" % opts.configure_command) +if opts.clean: + f.write("make clean || exit 125\n") +if opts.skip_build_errors: + build_err = 125 +else: + build_err = 1 +f.write("%s || exit %u\n" % (opts.build_command, build_err)) +f.write("%s || exit 1\n" % opts.test_command) +f.write("exit 0\n") +f.close() + + +def cleanup(): + run_cmd("git bisect reset", dir=gitroot) + os.unlink(f.name) + sys.exit(-1) + + +# run bisect +ret = -1 +try: + run_cmd("git bisect reset", dir=gitroot, show=False, checkfail=False) + run_cmd("git bisect start %s %s --" % (opts.bad, opts.good), dir=gitroot) + ret = run_cmd("git bisect run bash %s" % f.name, dir=gitroot, show=True, checkfail=False) +except KeyboardInterrupt: + print("Cleaning up") + cleanup() +except Exception as reason: + print("Failed bisect: %s" % reason) + cleanup() + +run_cmd("git bisect reset", dir=gitroot) +os.unlink(f.name) +sys.exit(ret) diff --git a/script/check-shell-scripts.sh b/script/check-shell-scripts.sh new file mode 100755 index 0000000..f4f48a3 --- /dev/null +++ b/script/check-shell-scripts.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +if [ $# -lt 1 ]; then + echo "Usage: $(basename "${0}") DIR [SEVERITY]" + exit 1 +fi + +DIR="${1}" +SEVERITY="${2:-error}" + +shfmt -f "${DIR}" | + grep -v -E "(bootstrap|third_party)" | + xargs shellcheck \ + --shell=sh \ + --external-sources \ + --check-sourced \ + --format=gcc \ + --severity="${SEVERITY}" + +exit $? diff --git a/script/clean-source-tree.sh b/script/clean-source-tree.sh new file mode 100755 index 0000000..8d7f759 --- /dev/null +++ b/script/clean-source-tree.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# + +N=$(git clean -n | wc -l) +C=$(git diff --stat HEAD | wc -l) + +test x"$N" != x"0" && { + echo "The tree has $N new uncommitted files!!! see stderr" + echo "The tree has $N new uncommitted files!!!" >&2 + + echo "git clean -n" >&2 + git clean -n >&2 + + test x"$C" != x"0" && { + echo "git diff -p --stat HEAD" >&2 + git diff -p --stat HEAD >&2 + } + + exit 1 +} + +test x"$C" != x"0" && { + echo "The tree has uncommitted changes!!! see stderr" + echo "The tree has uncommitted changes!!!" >&2 + + echo "git diff -p --stat HEAD" >&2 + git diff -p --stat HEAD >&2 + + exit 1 +} + +echo "clean tree" +exit 0 diff --git a/script/codespell.sh b/script/codespell.sh new file mode 100755 index 0000000..60e0eba --- /dev/null +++ b/script/codespell.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Check code spelling + +if [ $# -lt 1 ]; then + echo "Usage: $(basename "${0}") DIR" + exit 1 +fi + +DIR="${1}" + +codespell "${DIR}" +ret=$? + +if [ ${ret} -ne 0 ]; then + echo + echo "Fix code spelling issues. If it detected false positives" \ + "please update .codespellignore." +fi + +exit ${ret} diff --git a/script/commit_mark.sh b/script/commit_mark.sh new file mode 100755 index 0000000..3de6ba7 --- /dev/null +++ b/script/commit_mark.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# add a autobuild message to the HEAD commit + +branch=$(git branch --contains HEAD | grep '^\* ' | sed -e 's/^\* //') + +if grep -q "^Autobuild\-User($branch): " "$1"; then + echo "Already marked as tested for $branch" + exit 0 +fi + +fullname=$(getent passwd $USER | cut -d: -f5 | cut -d',' -f1) +mailaddr=$(git config user.email) +if test -z "$mailaddr"; then + mailaddr="$USER@samba.org" +fi +cat <<EOF >>"$1" + +Autobuild-User($branch): $fullname <$mailaddr> +Autobuild-Date($branch): $(date) on $(hostname) +EOF +exit 0 diff --git a/script/compare_cc_results.py b/script/compare_cc_results.py new file mode 100755 index 0000000..9bf24ad --- /dev/null +++ b/script/compare_cc_results.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Compare the results of native and cross-compiled configure tests + +The compared files are called "default.cache.py" and are generated in +bin/c4che/. + +USAGE: compare_cc_results.py CONFIG_1 CONFIG_2 [CONFIG_3 [CONFIG_4 ...]] +""" +import sys +import difflib + +exceptions = [ + 'BUILD_DIRECTORY', 'SELFTEST_PREFIX', 'defines', + 'CROSS_COMPILE', 'CROSS_ANSWERS', 'CROSS_EXECUTE', + 'LIBSOCKET_WRAPPER_SO_PATH', + 'LIBNSS_WRAPPER_SO_PATH', + 'LIBPAM_WRAPPER_SO_PATH', + 'PAM_SET_ITEMS_SO_PATH', + 'LIBUID_WRAPPER_SO_PATH', + 'LIBRESOLV_WRAPPER_SO_PATH', +] + +if len(sys.argv) < 3: + print(__doc__) + sys.exit(1) + +base_lines = list() +base_fname = '' + +found_diff = False + +for fname in sys.argv[1:]: + lines = list() + f = open(fname, 'r') + for line in f: + if line.startswith("cfg_files ="): + # waf writes configuration files as absolute paths + continue + if len(line.split('=', 1)) == 2: + key = line.split('=', 1)[0].strip() + value = line.split('=', 1)[1].strip() + if key in exceptions: + continue + # using waf with python 3.4 seems to randomly sort dict keys + # we can't modify the waf code but we can fake a dict value + # string representation as if it were sorted. python 3.6.5 + # doesn't seem to suffer from this behaviour + if value.startswith('{'): + import ast + amap = ast.literal_eval(value) + fakeline = "" + for k in sorted(amap.keys()): + if not len(fakeline) == 0: + fakeline = fakeline + ", " + fakeline = fakeline + '\'' + k + '\': \'' + amap[k] + '\'' + line = key + ' = {' + fakeline + '}' + lines.append(line) + f.close() + if base_fname: + diff = list(difflib.unified_diff(base_lines, lines, base_fname, fname)) + if diff: + print('configuration files %s and %s do not match' % (base_fname, fname)) + for l in diff: + sys.stdout.write(l) + found_diff = True + else: + base_fname = fname + base_lines = lines + +if found_diff: + sys.exit(1) diff --git a/script/configure_check_unused.pl b/script/configure_check_unused.pl new file mode 100755 index 0000000..52d8dee --- /dev/null +++ b/script/configure_check_unused.pl @@ -0,0 +1,124 @@ +#!/usr/bin/perl +# Script that finds macros in a configure script that are not +# used in a set of C files. +# Copyright Jelmer Vernooij <jelmer@samba.org>, GPL +# +# Usage: ./$ARGV[0] configure.in [c-files...] + +use strict; + +sub autoconf_parse($$$$) +{ + my $in = shift; + my $defines = shift; + my $functions = shift; + my $headers = shift; + + open(IN, $in) or die("Can't open $in"); + + my $ln = 0; + + foreach(<IN>) { + $ln++; + + if(/AC_DEFINE\(([^,]+),/) { + $defines->{$1} = "$in:$ln"; + } + + if(/AC_CHECK_FUNCS\(\[*(.[^],)]+)/) { + foreach(split / /, $1) { + $functions->{$_} = "$in:$ln"; + } + } + + if(/AC_CHECK_FUNC\(([^,)]+)/) { + $functions->{$1} = "$in:$ln"; + } + + if(/AC_CHECK_HEADERS\(\[*([^],)]+)/) { + foreach(split / /, $1) { + $headers->{$_} = "$in:$ln"; + } + } + + if(/AC_CHECK_HEADER\(([^,)]+)/) { + $headers->{$1} = "$in:$ln"; + } + + if(/sinclude\(([^,]+)\)/) { + autoconf_parse($1, $defines, $functions, $headers); + } + } + + close IN; +} + +# Return the symbols and headers used by a C file +sub cfile_parse($$$) +{ + my $in = shift; + my $symbols = shift; + my $headers = shift; + + open(FI, $in) or die("Can't open $in"); + my $ln = 0; + my $line; + while($line = <FI>) { + $ln++; + $_ = $line; + if (/\#([ \t]*)include ["<]([^">]+)/) { + $headers->{$2} = "$in:$ln"; + } + + $_ = $line; + while(/([A-Za-z0-9_]+)/g) { + $symbols->{$1} = "$in:$ln"; + } + } + close FI; +} + +my %ac_defines = (); +my %ac_func_checks = (); +my %ac_headers = (); +my %symbols = (); +my %headers = (); + +if (scalar(@ARGV) <= 1) { + print("Usage: configure_find_unused.pl configure.in [CFILE...]\n"); + exit 0; +} + +autoconf_parse(shift(@ARGV), \%ac_defines, \%ac_func_checks, \%ac_headers); +cfile_parse($_, \%symbols, \%headers) foreach(@ARGV); + +(keys %ac_defines) or warn("No defines found in configure.in file, parse error?"); + +foreach (keys %ac_defines) { + if (not defined($symbols{$_})) { + print "$ac_defines{$_}: Autoconf-defined $_ is unused\n"; + } +} + +(keys %ac_func_checks) or warn("No function checks found in configure.in file, parse error?"); + +foreach (keys %ac_func_checks) { + my $def = "HAVE_".uc($_); + if (not defined($symbols{$_})) { + print "$ac_func_checks{$_}: Autoconf-checked function `$_' is unused\n"; + } elsif (not defined($symbols{$def})) { + print "$ac_func_checks{$_}: Autoconf-define `$def' for function `$_' is unused\n"; + } +} + +(keys %ac_headers) or warn("No headers found in configure.in file, parse error?"); + +foreach (keys %ac_headers) { + my $def = "HAVE_".uc($_); + $def =~ s/[\/\.]/_/g; + if (not defined($headers{$_})) { + print "$ac_headers{$_}: Autoconf-checked header `$_' is unused\n"; + } elsif (not defined($symbols{$def})) { + print "$ac_headers{$_}: Autoconf-define `$def' for header `$_' is unused\n"; + } +} diff --git a/script/ctdb-import.msg-filter.sh b/script/ctdb-import.msg-filter.sh new file mode 100755 index 0000000..107d736 --- /dev/null +++ b/script/ctdb-import.msg-filter.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# + +set -e +set -u + +cat - +echo "" +echo "(This used to be ctdb commit ${GIT_COMMIT})" + +exit 0 diff --git a/script/ctdb-import.tree-filter.sh b/script/ctdb-import.tree-filter.sh new file mode 100755 index 0000000..00b6255 --- /dev/null +++ b/script/ctdb-import.tree-filter.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# + +set -e +set -u + +lo=$(find -mindepth 1 -maxdepth 1) +for o in $lo; do + mkdir -p ctdb + mv $o ctdb/ +done + +exit 0 diff --git a/script/ctdb-import.txt b/script/ctdb-import.txt new file mode 100644 index 0000000..621b24e --- /dev/null +++ b/script/ctdb-import.txt @@ -0,0 +1,5 @@ +ctdb-import.git$ git filter-branch \ + --tree-filter /path/to/ctdb-import.tree-filter.sh \ + --msg-filter /path/to/ctdb-import.msg-filter.sh \ + HEAD + diff --git a/script/find_python.sh b/script/find_python.sh new file mode 100755 index 0000000..5ef8368 --- /dev/null +++ b/script/find_python.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +if [ $# -lt 1 ]; then + echo "$0: <installdir>" + exit 1 +fi + +installdir=$1 +exit $(find ${installdir} -name \*.py | wc -l) diff --git a/script/findstatic.pl b/script/findstatic.pl new file mode 100755 index 0000000..43a4916 --- /dev/null +++ b/script/findstatic.pl @@ -0,0 +1,70 @@ +#!/usr/bin/perl -w +# find a list of fns and variables in the code that could be static +# usually called with something like this: +# findstatic.pl `find . -name "*.o"` +# Andrew Tridgell <tridge@samba.org> + +use strict; + +# use nm to find the symbols +my($saved_delim) = $/; +undef $/; +my($syms) = `nm -o @ARGV`; +$/ = $saved_delim; + +my(@lines) = split(/\n/s, $syms); + +my(%def); +my(%undef); +my(%stype); + +my(%typemap) = ( + "T" => "function", + "C" => "uninitialised variable", + "D" => "initialised variable" + ); + + +# parse the symbols into defined and undefined +for (my($i)=0; $i <= $#{@lines}; $i++) { + my($line) = $lines[$i]; + if ($line =~ /(.*):[a-f0-9]* ([TCD]) (.*)/) { + my($fname) = $1; + my($symbol) = $3; + push(@{$def{$fname}}, $symbol); + $stype{$symbol} = $2; + } + if ($line =~ /(.*):\s* U (.*)/) { + my($fname) = $1; + my($symbol) = $2; + push(@{$undef{$fname}}, $symbol); + } +} + +# look for defined symbols that are never referenced outside the place they +# are defined +foreach my $f (keys %def) { + print "Checking $f\n"; + my($found_one) = 0; + foreach my $s (@{$def{$f}}) { + my($found) = 0; + foreach my $f2 (keys %undef) { + if ($f2 ne $f) { + foreach my $s2 (@{$undef{$f2}}) { + if ($s2 eq $s) { + $found = 1; + $found_one = 1; + } + } + } + } + if ($found == 0) { + my($t) = $typemap{$stype{$s}}; + print " '$s' is unique to $f ($t)\n"; + } + } + if ($found_one == 0) { + print " all symbols in '$f' are unused (main program?)\n"; + } +} + diff --git a/script/generate_param.py b/script/generate_param.py new file mode 100644 index 0000000..50f2d12 --- /dev/null +++ b/script/generate_param.py @@ -0,0 +1,431 @@ +# Unix SMB/CIFS implementation. +# Copyright (C) 2014 Catalyst.Net Ltd +# +# Auto generate param_functions.c +# +# ** NOTE! The following LGPL license applies to the ldb +# ** library. This does NOT imply that all of Samba is released +# ** under the LGPL +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see <http://www.gnu.org/licenses/>. +# + +import os +import xml.etree.ElementTree as ET +import optparse + +# parse command line arguments +parser = optparse.OptionParser() +parser.add_option("-f", "--file", dest="filename", + help="input file", metavar="FILE") +parser.add_option("-o", "--output", dest="output", + help='output file', metavar="FILE") +parser.add_option("--mode", type="choice", metavar="<FUNCTIONS|S3PROTO|LIBPROTO|PARAMDEFS|PARAMTABLE>", + choices=["FUNCTIONS", "S3PROTO", "LIBPROTO", "PARAMDEFS", "PARAMTABLE"], default="FUNCTIONS") +parser.add_option("--scope", metavar="<GLOBAL|LOCAL>", + choices=["GLOBAL", "LOCAL"], default="GLOBAL") + +(options, args) = parser.parse_args() + +if options.filename is None: + parser.error("No input file specified") +if options.output is None: + parser.error("No output file specified") + + +def iterate_all(path): + """Iterate and yield all the parameters. + + :param path: path to parameters xml file + """ + + try: + with open(path, 'r') as p: + out = p.read() + except IOError as e: + raise Exception("Error opening parameters file") + + # parse the parameters xml file + root = ET.fromstring(out) + for parameter in root: + name = parameter.attrib.get("name") + param_type = parameter.attrib.get("type") + context = parameter.attrib.get("context") + func = parameter.attrib.get("function") + synonym = parameter.attrib.get("synonym") + removed = parameter.attrib.get("removed") + generated = parameter.attrib.get("generated_function") + handler = parameter.attrib.get("handler") + enumlist = parameter.attrib.get("enumlist") + deprecated = parameter.attrib.get("deprecated") + synonyms = parameter.findall('synonym') + + if removed == "1": + continue + + constant = parameter.attrib.get("constant") + substitution = parameter.attrib.get("substitution") + parm = parameter.attrib.get("parm") + if name is None or param_type is None or context is None: + raise Exception("Error parsing parameter: " + name) + if func is None: + func = name.replace(" ", "_").lower() + if enumlist is None: + enumlist = "NULL" + if handler is None: + handler = "NULL" + yield {'name': name, + 'type': param_type, + 'context': context, + 'function': func, + 'constant': (constant == '1'), + 'substitution': (substitution == '1'), + 'parm': (parm == '1'), + 'synonym' : synonym, + 'generated' : generated, + 'enumlist' : enumlist, + 'handler' : handler, + 'deprecated' : deprecated, + 'synonyms' : synonyms } + + +# map doc attributes to a section of the generated function +context_dict = {"G": "_GLOBAL", "S": "_LOCAL"} +param_type_dict = { + "boolean" : "_BOOL", + "list" : "_LIST", + "string" : "_STRING", + "integer" : "_INTEGER", + "enum" : "_INTEGER", + "char" : "_CHAR", + "boolean-auto" : "_INTEGER", + "cmdlist" : "_LIST", + "bytes" : "_INTEGER", + "octal" : "_INTEGER", + "ustring" : "_STRING", + } + + +def generate_functions(path_in, path_out): + f = open(path_out, 'w') + try: + f.write('/* This file was automatically generated by generate_param.py. DO NOT EDIT */\n\n') + for parameter in iterate_all(options.filename): + # filter out parameteric options + if ':' in parameter['name']: + continue + if parameter['synonym'] == "1": + continue + if parameter['generated'] == "0": + continue + + output_string = "FN" + temp = context_dict.get(parameter['context']) + if temp is None: + raise Exception(parameter['name'] + " has an invalid context " + parameter['context']) + output_string += temp + if parameter['type'] == "string" or parameter['type'] == "ustring": + if parameter['substitution']: + output_string += "_SUBSTITUTED" + else: + output_string += "_CONST" + if parameter['parm']: + output_string += "_PARM" + temp = param_type_dict.get(parameter['type']) + if temp is None: + raise Exception(parameter['name'] + " has an invalid param type " + parameter['type']) + output_string += temp + f.write(output_string + "(" + parameter['function'] + ", " + parameter['function'] + ')\n') + finally: + f.close() + + +mapping = { + 'boolean' : 'bool ', + 'string' : 'char *', + 'integer' : 'int ', + 'char' : 'char ', + 'list' : 'const char **', + 'enum' : 'int ', + 'boolean-auto' : 'int ', + 'cmdlist' : 'const char **', + 'bytes' : 'int ', + 'octal' : 'int ', + 'ustring' : 'char *', + } + + +def make_s3_param_proto(path_in, path_out): + file_out = open(path_out, 'w') + try: + file_out.write('/* This file was automatically generated by generate_param.py. DO NOT EDIT */\n\n') + header = get_header(path_out) + file_out.write("#ifndef %s\n" % header) + file_out.write("#define %s\n\n" % header) + file_out.write("struct share_params;\n") + file_out.write("struct loadparm_substitution;\n") + for parameter in iterate_all(path_in): + # filter out parameteric options + if ':' in parameter['name']: + continue + if parameter['synonym'] == "1": + continue + if parameter['generated'] == "0": + continue + + output_string = "" + param_type = mapping.get(parameter['type']) + if param_type is None: + raise Exception(parameter['name'] + " has an invalid context " + parameter['context']) + output_string += param_type + output_string += "lp_%s" % parameter['function'] + + param = None + if parameter['parm']: + param = "const struct share_params *p" + else: + param = "int" + + if parameter['type'] == 'string' or parameter['type'] == 'ustring': + if parameter['substitution']: + if parameter['context'] == 'G': + output_string += '(TALLOC_CTX *ctx, const struct loadparm_substitution *lp_sub);\n' + elif parameter['context'] == 'S': + output_string += '(TALLOC_CTX *ctx, const struct loadparm_substitution *lp_sub, %s);\n' % param + else: + raise Exception(parameter['name'] + " has an invalid param type " + parameter['type']) + else: + if parameter['context'] == 'G': + output_string = 'const ' + output_string + '(void);\n' + elif parameter['context'] == 'S': + output_string = 'const ' + output_string + '(%s);\n' % param + else: + raise Exception(parameter['name'] + " has an invalid param type " + parameter['type']) + else: + if parameter['context'] == 'G': + output_string += '(void);\n' + elif parameter['context'] == 'S': + output_string += '(%s);\n' % param + else: + raise Exception(parameter['name'] + " has an invalid param type " + parameter['type']) + + file_out.write(output_string) + + file_out.write("\n#endif /* %s */\n\n" % header) + finally: + file_out.close() + + +def make_lib_proto(path_in, path_out): + file_out = open(path_out, 'w') + try: + file_out.write('/* This file was automatically generated by generate_param.py. DO NOT EDIT */\n\n') + for parameter in iterate_all(path_in): + # filter out parameteric options + if ':' in parameter['name']: + continue + if parameter['synonym'] == "1": + continue + if parameter['generated'] == "0": + continue + + output_string = "" + param_type = mapping.get(parameter['type']) + if param_type is None: + raise Exception(parameter['name'] + " has an invalid context " + parameter['context']) + output_string += param_type + + output_string += "lpcfg_%s" % parameter['function'] + + if parameter['type'] == 'string' or parameter['type'] == 'ustring': + if parameter['substitution']: + if parameter['context'] == 'G': + output_string += '(struct loadparm_context *, const struct loadparm_substitution *lp_sub, TALLOC_CTX *ctx);\n' + elif parameter['context'] == 'S': + output_string += '(struct loadparm_service *, struct loadparm_service *, TALLOC_CTX *ctx);\n' + else: + raise Exception(parameter['name'] + " has an invalid context " + parameter['context']) + else: + if parameter['context'] == 'G': + output_string = 'const ' + output_string + '(struct loadparm_context *);\n' + elif parameter['context'] == 'S': + output_string = 'const ' + output_string + '(struct loadparm_service *, struct loadparm_service *);\n' + else: + raise Exception(parameter['name'] + " has an invalid param type " + parameter['type']) + else: + if parameter['context'] == 'G': + output_string += '(struct loadparm_context *);\n' + elif parameter['context'] == 'S': + output_string += '(struct loadparm_service *, struct loadparm_service *);\n' + else: + raise Exception(parameter['name'] + " has an invalid param type " + parameter['type']) + + file_out.write(output_string) + finally: + file_out.close() + + +def get_header(path): + header = os.path.basename(path).upper() + header = header.replace(".", "_").replace("\\", "_").replace("-", "_") + return "__%s__" % header + + +def make_param_defs(path_in, path_out, scope): + file_out = open(path_out, 'w') + try: + file_out.write('/* This file was automatically generated by generate_param.py. DO NOT EDIT */\n\n') + header = get_header(path_out) + file_out.write("#ifndef %s\n" % header) + file_out.write("#define %s\n\n" % header) + if scope == "GLOBAL": + file_out.write("/**\n") + file_out.write(" * This structure describes global (ie., server-wide) parameters.\n") + file_out.write(" */\n") + file_out.write("struct loadparm_global \n") + file_out.write("{\n") + file_out.write("\tTALLOC_CTX *ctx; /* Context for talloced members */\n") + elif scope == "LOCAL": + file_out.write("/**\n") + file_out.write(" * This structure describes a single service.\n") + file_out.write(" */\n") + file_out.write("struct loadparm_service \n") + file_out.write("{\n") + file_out.write("\tbool autoloaded;\n") + + for parameter in iterate_all(path_in): + # filter out parameteric options + if ':' in parameter['name']: + continue + if parameter['synonym'] == "1": + continue + + if (scope == "GLOBAL" and parameter['context'] != "G" or + scope == "LOCAL" and parameter['context'] != "S"): + continue + + output_string = "\t" + param_type = mapping.get(parameter['type']) + if param_type is None: + raise Exception(parameter['name'] + " has an invalid context " + parameter['context']) + output_string += param_type + + output_string += " %s;\n" % parameter['function'] + file_out.write(output_string) + + file_out.write("LOADPARM_EXTRA_%sS\n" % scope) + file_out.write("};\n") + file_out.write("\n#endif /* %s */\n\n" % header) + finally: + file_out.close() + + +type_dict = { + "boolean" : "P_BOOL", + "boolean-rev" : "P_BOOLREV", + "boolean-auto" : "P_ENUM", + "list" : "P_LIST", + "string" : "P_STRING", + "integer" : "P_INTEGER", + "enum" : "P_ENUM", + "char" : "P_CHAR", + "cmdlist" : "P_CMDLIST", + "bytes" : "P_BYTES", + "octal" : "P_OCTAL", + "ustring" : "P_USTRING", + } + + +def make_param_table(path_in, path_out): + file_out = open(path_out, 'w') + try: + file_out.write('/* This file was automatically generated by generate_param.py. DO NOT EDIT */\n\n') + header = get_header(path_out) + file_out.write("#ifndef %s\n" % header) + file_out.write("#define %s\n\n" % header) + + file_out.write("struct parm_struct parm_table[] = {\n") + + for parameter in iterate_all(path_in): + # filter out parameteric options + if ':' in parameter['name']: + continue + if parameter['context'] == 'G': + p_class = "P_GLOBAL" + else: + p_class = "P_LOCAL" + + p_type = type_dict.get(parameter['type']) + + if parameter['context'] == 'G': + temp = "GLOBAL" + else: + temp = "LOCAL" + offset = "%s_VAR(%s)" % (temp, parameter['function']) + + enumlist = parameter['enumlist'] + handler = parameter['handler'] + synonym = parameter['synonym'] + deprecated = parameter['deprecated'] + flags_list = [] + if synonym == "1": + flags_list.append("FLAG_SYNONYM") + if deprecated == "1": + flags_list.append("FLAG_DEPRECATED") + flags = "|".join(flags_list) + synonyms = parameter['synonyms'] + + file_out.write("\t{\n") + file_out.write("\t\t.label\t\t= \"%s\",\n" % parameter['name']) + file_out.write("\t\t.type\t\t= %s,\n" % p_type) + file_out.write("\t\t.p_class\t= %s,\n" % p_class) + file_out.write("\t\t.offset\t\t= %s,\n" % offset) + file_out.write("\t\t.special\t= %s,\n" % handler) + file_out.write("\t\t.enum_list\t= %s,\n" % enumlist) + if flags != "": + file_out.write("\t\t.flags\t\t= %s,\n" % flags) + file_out.write("\t},\n") + + if synonyms is not None: + # for synonyms, we only list the synonym flag: + flags = "FLAG_SYNONYM" + for syn in synonyms: + file_out.write("\t{\n") + file_out.write("\t\t.label\t\t= \"%s\",\n" % syn.text) + file_out.write("\t\t.type\t\t= %s,\n" % p_type) + file_out.write("\t\t.p_class\t= %s,\n" % p_class) + file_out.write("\t\t.offset\t\t= %s,\n" % offset) + file_out.write("\t\t.special\t= %s,\n" % handler) + file_out.write("\t\t.enum_list\t= %s,\n" % enumlist) + if flags != "": + file_out.write("\t\t.flags\t\t= %s,\n" % flags) + file_out.write("\t},\n") + + file_out.write("\n\t{ .label = NULL }\n") + file_out.write("};\n") + file_out.write("\n#endif /* %s */\n\n" % header) + finally: + file_out.close() + + +if options.mode == 'FUNCTIONS': + generate_functions(options.filename, options.output) +elif options.mode == 'S3PROTO': + make_s3_param_proto(options.filename, options.output) +elif options.mode == 'LIBPROTO': + make_lib_proto(options.filename, options.output) +elif options.mode == 'PARAMDEFS': + make_param_defs(options.filename, options.output, options.scope) +elif options.mode == 'PARAMTABLE': + make_param_table(options.filename, options.output) diff --git a/script/git-hooks/check-trailing-whitespace b/script/git-hooks/check-trailing-whitespace new file mode 100755 index 0000000..4dc1a6d --- /dev/null +++ b/script/git-hooks/check-trailing-whitespace @@ -0,0 +1,17 @@ +#!/bin/sh + +git diff-index --cached --check HEAD -- :/*.[ch] :/*.p[ylm] + +if [ $? != 0 ]; then + echo + echo "The commit failed because it seems to introduce trailing whitespace" + echo "into C, Perl, or Python code." + echo + echo "If you are sure you want to do this, repeat the commit with the " + echo "--no-verify, like this:" + echo + echo " git commit --no-verify" + exit 1 +fi + +exit 0 diff --git a/script/git-hooks/pre-commit-hook b/script/git-hooks/pre-commit-hook new file mode 100755 index 0000000..1bcb000 --- /dev/null +++ b/script/git-hooks/pre-commit-hook @@ -0,0 +1,17 @@ +#!/bin/sh + +set -eu + +gitdir=$(git rev-parse --show-toplevel) +if [ $? -ne 0 ]; then + echo "git rev-parse --show-toplevel failed" + exit 1 +fi + +if [ ! -f ${gitdir}/script/git-hooks/pre-commit-script ]; then + exit 0 +fi + +${gitdir}/script/git-hooks/pre-commit-script || exit $? + +exit 0 diff --git a/script/git-hooks/pre-commit-script b/script/git-hooks/pre-commit-script new file mode 100755 index 0000000..8adb01c --- /dev/null +++ b/script/git-hooks/pre-commit-script @@ -0,0 +1,19 @@ +#!/bin/sh + +set -eu + +# +# make emacs/magit work, cf +# https://github.com/magit/magit/issues/3419 +# +unset GIT_LITERAL_PATHSPECS + +gitdir=$(git rev-parse --show-toplevel) +if [ $? -ne 0 ]; then + echo "git rev-parse --show-toplevel failed" + exit 1 +fi + +${gitdir}/script/git-hooks/check-trailing-whitespace || exit $? + +exit 0 diff --git a/script/identity_cc.sh b/script/identity_cc.sh new file mode 100755 index 0000000..8b5118e --- /dev/null +++ b/script/identity_cc.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +#An "identity cross-execute" script +#It can be used for testing the cross-build infrastructure +#as follows: +#./configure --cross-compile --cross-execute=./script/identity_cc.sh +#If the build is actually a native build, then the configuration +#result should be just like running ./configure without --cross-compile. + +eval "$@" diff --git a/script/release.sh b/script/release.sh new file mode 100755 index 0000000..b144f74 --- /dev/null +++ b/script/release.sh @@ -0,0 +1,1275 @@ +#!/bin/bash +# make a release of Samba or a library + +LC_ALL=C +export LC_ALL +LANG=C +export LANG +LANGUAGE=C +export LANGUAGE + +set -u +set -e +umask 0022 + +CONF_REPO_URL="ssh://git.samba.org/data/git/samba.git" +CONF_UPLOAD_URL="samba-bugs@download-master.samba.org:/home/data/ftp/pub" +CONF_DOWNLOAD_URL="https://download.samba.org/pub" +CONF_HISTORY_URL="https://www.samba.org" + +test -d ".git" -o -r ".git" || { + echo "Run this script from the top-level directory in the" + echo "repository" + exit 1 +} + +usage() +{ + echo "Usage: script/release.sh <PRODUCT> <COMMAND>" + echo "" + echo "PRODUCT: ldb, talloc, tevent, tdb, samba-rc, samba-stable" + echo "COMMAND: fullrelease, create, push, upload, announce" + echo "" + return 0 +} + +test -x "script/release.sh" || { + usage + echo "Run this script from the top-level directory in the" + echo "repository: as 'script/release.sh'" + exit 1 +} + +check_args() +{ + local cmd="$1" + local got_args="$2" + local take_args="$3" + + test x"${got_args}" = x"${take_args}" || { + usage + echo "cmd[${cmd}] takes ${take_args} instead of ${got_args}" + return 1 + } + + return 0 +} + +min_args() +{ + local cmd="$1" + local got_args="$2" + local min_args="$3" + + test "${got_args}" -ge "${min_args}" || { + usage + echo "cmd[${cmd}] takes at least ${min_args} instead of ${got_args}" + return 1 + } + + return 0 +} + +min_args "$0" "$#" "2" + +product="$1" +globalcmd="$2" +shift 2 +oldtagname="" +tagname="" +patchfile="" +cmds="" +next_cmd="" + +require_tagname() +{ + min_args "${FUNCNAME}" "$#" "1" || return 1 + local cmd="$1" + + test -n "${tagname}" || { + echo "cmd[${cmd}] requires '\${tagname}' variable to be set" + return 1 + } + + local name=$(echo "${tagname}" | cut -d '-' -f1) + test x"${name}" = x"${productbase}" || { + echo "Invalid tagname[${tgzname}]" + return 1 + } + + return 0 +} + +cmd_allowed() +{ + min_args "${FUNCNAME}" "$#" "2" || return 1 + local cmd="$1" + shift 1 + + echo "$@" | grep -q "\<${cmd}\>" || { + return 1 + } + + return 0 +} + +verify_samba_rc() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + + test -f VERSION || { + echo "VERSION doesn't exist" + return 1 + } + + grep -q 'SAMBA_VERSION_IS_GIT_SNAPSHOT=no' VERSION || { + echo "SAMBA_VERSION_IS_GIT_SNAPSHOT is not 'no'" + return 1 + } + + grep -q '^SAMBA_VERSION_RC_RELEASE=' VERSION || { + echo "SAMBA_VERSION_RC_RELEASE= missing" + return 1 + } + + grep -q '^SAMBA_VERSION_RC_RELEASE=$' VERSION && { + echo "SAMBA_VERSION_RC_RELEASE= missing the rc version" + return 1 + } + + return 0 +} + +load_samba_stable_versions() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + + test -n "${version-}" && { + return 0 + } + + local SAMBA_VERSION_MAJOR=$(grep '^SAMBA_VERSION_MAJOR=' VERSION | cut -d '=' -f2 | xargs) + local SAMBA_VERSION_MINOR=$(grep '^SAMBA_VERSION_MINOR=' VERSION | cut -d '=' -f2 | xargs) + local SAMBA_VERSION_RELEASE=$(grep '^SAMBA_VERSION_RELEASE=' VERSION | cut -d '=' -f2 | xargs) + + version="${SAMBA_VERSION_MAJOR}.${SAMBA_VERSION_MINOR}.${SAMBA_VERSION_RELEASE}" + tagname="${productbase}-${version}" + + test ${SAMBA_VERSION_RELEASE} -gt 0 || { + return 0 + } + + oldversion="${SAMBA_VERSION_MAJOR}.${SAMBA_VERSION_MINOR}.$(expr ${SAMBA_VERSION_RELEASE} - 1)" + oldtagname="${productbase}-${oldversion}" + patchfile="${productbase}-${oldversion}-${version}.diffs" + + return 0 +} + +verify_samba_stable() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + + test -f VERSION || { + echo "VERSION doesn't exist" + return 1 + } + + grep -q 'SAMBA_VERSION_IS_GIT_SNAPSHOT=no' VERSION || { + echo "SAMBA_VERSION_IS_GIT_SNAPSHOT is not 'no'" + return 1 + } + + local VARS="" + VARS="${VARS} SAMBA_VERSION_REVISION" + VARS="${VARS} SAMBA_VERSION_TP_RELEASE" + VARS="${VARS} SAMBA_VERSION_ALPHA_RELEASE" + VARS="${VARS} SAMBA_VERSION_BETA_RELEASE" + VARS="${VARS} SAMBA_VERSION_PRE_RELEASE" + VARS="${VARS} SAMBA_VERSION_RC_RELEASE" + VARS="${VARS} SAMBA_VERSION_RELEASE_NICKNAME" + VARS="${VARS} SAMBA_VERSION_VENDOR_SUFFIX" + VARS="${VARS} SAMBA_VERSION_VENDOR_PATCH" + for var in ${VARS}; do + grep -q "^${var}" VERSION && { + grep -q "^${var}=$" VERSION || { + echo "${var} found in stable version" + return 1 + } + } + done + + load_samba_stable_versions + + test -n "${oldtagname}" || { + return 0 + } + + local verify_out="${TMPDIR}/verify-${oldtagname}.out" + + echo "Verifying oldtagname: ${oldtagname}" + + git tag -v "${oldtagname}" >${verify_out} 2>&1 || { + echo "failed to verify old tag[${oldtagname}]" + echo "" + cat "${verify_out}" + return 1 + } + + grep -q "${GPG_KEYID}" "${verify_out}" || { + echo "oldtagname[${oldtagname}] was not generated with GPG_KEYID[${GPG_KEYID}]!" + echo "" + cat "${verify_out}" + return 1 + } + + echo "Verifying ${oldtagname}.tar.gz and ${oldtagname}.tar.asc" + + test -f "${oldtagname}.tar.gz" || { + echo "${oldtagname}.tar.gz does not exist" + return 1 + } + + test -f "${oldtagname}.tar.asc" || { + echo "${oldtagname}.tar.asc does not exist" + return 1 + } + + zcat "${oldtagname}.tar.gz" | gpg --verify "${oldtagname}.tar.asc" - 2>${verify_out} || { + echo "Failed to verify ${oldtagname}.tar.asc" + return 1 + } + + grep -q "${GPG_KEYID}" "${verify_out}" || { + echo "${oldtagname}.tar.asc was not generated with GPG_KEYID[${GPG_KEYID}]!" + echo "" + cat "${verify_out}" + return 1 + } + + return 0 +} + +verify_release() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + + test -n "${verify_fn}" || { + echo "verify_fn variable empty" + return 1 + } + + echo "Running ${verify_fn}" + ${verify_fn} +} + +create_release() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + + echo "Releasing product ${product}" + + test -n "${tagname}" && { + git tag -l "${tagname}" | grep -q "${tagname}" && { + echo "tagname[${tagname}] already exist" + return 1 + } + + local _tgzname="${tagname}.tar.gz" + test -e "${_tgzname}" && { + echo "_tgzname[${_tgzname}] already exist" + return 1 + } + } + + echo "Building release tarball" + local tgzname=$(make dist 2>&1 | grep ^Created | cut -d' ' -f2) + test -f "${tgzname}" || { + echo "Failed to create tarball" + return 1 + } + CLEANUP_FILES="${CLEANUP_FILES} ${tgzname}" + + local name=$(echo "${tgzname}" | cut -d '-' -f1) + test x"${name}" = x"${productbase}" || { + echo "Invalid tgzname[${tgzname}]" + return 1 + } + + local _tagname=$(basename ${tgzname} .tar.gz) + test -n "${tagname}" && { + test x"${_tagname}" = x"${tagname}" || { + echo "Invalid tgzname[${tgzname}]" + return 1 + } + } + tagname="${_tagname}" + + local tarname=$(basename ${tgzname} .gz) + echo "Tarball: ${tarname}" + gunzip -f ${tgzname} || { + echo "Failed to decompress tarball ${tarname}" + return 1 + } + test -f "${tarname}" || { + echo "Failed to decompress tarball ${tarname}" + return 1 + } + CLEANUP_FILES="${CLEANUP_FILES} ${tarname}" + + # tagname is global + echo "Tagging as ${tagname}" + git tag -u ${GPG_KEYID} -s "${tagname}" -m "${productbase}: tag release ${tagname}" || { + return 1 + } + CLEANUP_TAGS="${CLEANUP_TAGS} ${tagname}" + + echo "Signing ${tarname} => ${tarname}.asc" + rm -f "${tarname}.asc" + gpg --default-key "${GPG_KEYID}" --detach-sign --armor ${tarname} || { + return 1 + } + test -f "${tarname}.asc" || { + echo "Failed to create signature ${tarname}.asc" + return 1 + } + CLEANUP_FILES="${CLEANUP_FILES} ${tarname}.asc" + echo "Compressing ${tarname} => ${tgzname}" + gzip -f -9 ${tarname} + test -f "${tgzname}" || { + echo "Failed to compress ${tgzname}" + return 1 + } + + return 0 +} + +patch_release() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + require_tagname "${FUNCNAME}" + + test -n "${patchfile}" || { + return 0 + } + + local oldpwd=$(pwd) + echo "Generating ${patchfile}" + ( + set -e + set -u + pushd "${TMPDIR}" + tar xfz "${oldpwd}/${oldtagname}.tar.gz" + tar xfz "${oldpwd}/${tagname}.tar.gz" + diff -Npur "${oldtagname}/" "${tagname}/" >"${patchfile}" + popd + ) + CLEANUP_FILES="${CLEANUP_FILES} ${patchfile}" + mv "${TMPDIR}/${patchfile}" "${patchfile}" || { + echo "failed cmd[mv ${TMPDIR}/${patchfile} ${patchfile}]" + return 1 + } + + echo "Signing ${patchfile} => ${patchfile}.asc" + rm -f "${patchfile}.asc" + CLEANUP_FILES="${CLEANUP_FILES} ${patchfile}.asc" + gpg --default-key "${GPG_KEYID}" --detach-sign --armor ${patchfile} || { + return 1 + } + test -f "${patchfile}.asc" || { + echo "Failed to create signature ${patchfile}.asc" + return 1 + } + echo "Compressing ${patchfile} => ${patchfile}.gz" + CLEANUP_FILES="${CLEANUP_FILES} ${patchfile}.gz" + gzip -f -9 ${patchfile} + test -f "${patchfile}.gz" || { + echo "Failed to compress ${patchfile}.gz" + return 1 + } + + return 0 +} + +whatsnew_release() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + require_tagname "${FUNCNAME}" + + echo "extract ${tagname}.WHATSNEW.txt" + tar xf ${tagname}.tar.gz --to-stdout ${tagname}/WHATSNEW.txt >${tagname}.WHATSNEW.txt + CLEANUP_FILES="${CLEANUP_FILES} ${tagname}.WHATSNEW.txt" + + return 0 +} + +check_nopatch() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + require_tagname "${FUNCNAME}" + + local verify_out="${TMPDIR}/verify-${oldtagname}.out" + + echo "Verifying tagname: ${tagname}" + + git tag -v "${tagname}" >${verify_out} 2>&1 || { + echo "failed to verify tag[${tagname}]" + echo "" + cat "${verify_out}" + return 1 + } + grep -q "${GPG_KEYID}" "${verify_out}" || { + echo "tagname[${tagname}] was not generated with GPG_KEYID[${GPG_KEYID}]!" + echo "" + cat "${verify_out}" + return 1 + } + + echo "Verifying ${tagname}.tar.gz and ${tagname}.tar.asc" + + test -f "${tagname}.tar.gz" || { + echo "${tagname}.tar.gz does not exist" + return 1 + } + + test -f "${tagname}.tar.asc" || { + echo "${tagname}.tar.asc does not exist" + return 1 + } + + zcat "${tagname}.tar.gz" | gpg --verify "${tagname}.tar.asc" - 2>${verify_out} || { + echo "Failed to verify ${tagname}.tar.asc" + return 1 + } + grep -q "${GPG_KEYID}" "${verify_out}" || { + echo "${tagname}.tar.asc was not generated with GPG_KEYID[${GPG_KEYID}]!" + echo "" + cat "${verify_out}" + return 1 + } + + ls -la ${tagname}.* + + return 0 +} + +check_samba_stable() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + require_tagname "${FUNCNAME}" + + load_samba_stable_versions + + local verify_out="${TMPDIR}/verify-${oldtagname}.out" + + echo "Verifying tagname: ${tagname}" + + git tag -v "${tagname}" >${verify_out} 2>&1 || { + echo "failed to verify tag[${tagname}]" + echo "" + cat "${verify_out}" + return 1 + } + grep -q "${GPG_KEYID}" "${verify_out}" || { + echo "tagname[${tagname}] was not generated with GPG_KEYID[${GPG_KEYID}]!" + echo "" + cat "${verify_out}" + return 1 + } + + echo "Verifying ${tagname}.tar.gz and ${tagname}.tar.asc" + + test -f "${tagname}.tar.gz" || { + echo "${tagname}.tar.gz does not exist" + return 1 + } + + test -f "${tagname}.tar.asc" || { + echo "${tagname}.tar.asc does not exist" + return 1 + } + + zcat "${tagname}.tar.gz" | gpg --verify "${tagname}.tar.asc" - 2>${verify_out} || { + echo "Failed to verify ${tagname}.tar.asc" + return 1 + } + grep -q "${GPG_KEYID}" "${verify_out}" || { + echo "${tagname}.tar.asc was not generated with GPG_KEYID[${GPG_KEYID}]!" + echo "" + cat "${verify_out}" + return 1 + } + + test -n "${patchfile}" || { + ls -lart ${tagname}.* + return 0 + } + + echo "Verifying ${patchfile}.gz and ${patchfile}.asc" + + test -f "${patchfile}.gz" || { + echo "${patchfile}.gz does not exist" + return 1 + } + + test -f "${patchfile}.asc" || { + echo "${patchfile}.asc does not exist" + return 1 + } + + zcat "${patchfile}.gz" | gpg --verify "${patchfile}.asc" - 2>${verify_out} || { + echo "Failed to verify ${patchfile}.asc" + return 1 + } + grep -q "${GPG_KEYID}" "${verify_out}" || { + echo "${patchfile}.asc was not generated with GPG_KEYID[${GPG_KEYID}]!" + echo "" + cat "${verify_out}" + return 1 + } + + ls -lart ${tagname}.* ${patchfile}.* + return 0 +} + +check_release() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + + test -n "${check_fn}" || { + echo "check_fn variable empty" + return 1 + } + + echo "Running ${check_fn}" + ${check_fn} +} + +push_release() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + require_tagname "${FUNCNAME}" + + echo "Push git tag ${tagname} to '${repo_url}'" + git push "${repo_url}" "refs/tags/${tagname}:refs/tags/${tagname}" || { + return 1 + } + + return 0 +} + +upload_nopatch() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + require_tagname "${FUNCNAME}" + + echo "Upload ${tagname}.* to '${upload_url}'" + rsync -Pav --delay-updates ${tagname}.* "${upload_url}/" || { + return 1 + } + rsync ${upload_url}/${tagname}.* + + return 0 +} + +upload_samba_stable() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + require_tagname "${FUNCNAME}" + + load_samba_stable_versions + + local release_url="${upload_url}samba/stable/" + local patch_url="${upload_url}samba/patches/" + + echo "Upload ${tagname}.tar.* to '${release_url}'" + ls -lart ${tagname}.tar.* + rsync -Pav --delay-updates ${tagname}.tar.* "${release_url}/" || { + return 1 + } + rsync ${release_url}/${tagname}.tar.* + + test -n "${patchfile}" || { + return 0 + } + + echo "Upload ${patchfile}.* to '${patch_url}'" + ls -lart ${patchfile}.* + rsync -Pav --delay-updates ${patchfile}.* "${patch_url}/" || { + return 1 + } + rsync ${patch_url}/${patchfile}.* + + return 0 +} + +upload_release() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + + test -n "${upload_fn}" || { + echo "upload_fn variable empty" + return 1 + } + + echo "Running ${upload_fn}" + ${upload_fn} +} + +announcement_samba_rc() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + require_tagname "${FUNCNAME}" + + test -f "${tagname}.WHATSNEW.txt" || { + echo "${tagname}.WHATSNEW.txt does not exist" + return 1 + } + + local t="" + local version=$(echo "${tagname}" | sed -e 's!^samba-!!') + local href="#${version}" + local series=$(echo "${version}" | cut -d '.' -f1-2) + local rc=$(echo "${version}" | sed -e 's!.*rc\([0-9][0-9]*\)!\1!') + local rcname="${rc}th" + case "${rc}" in + 1) + rcname="first" + ;; + 2) + rcname="second" + ;; + 3) + rcname="third" + ;; + 4) + rcname="fourth" + ;; + 5) + rcname="fifth" + ;; + esac + + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.to.txt" + { + echo "samba-announce@lists.samba.org, samba@lists.samba.org, samba-technical@lists.samba.org" + } >announce.${tagname}.to.txt + + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.subject.txt" + { + echo "[Announce] Samba ${version} Available for Download" + } >announce.${tagname}.subject.txt + + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.mail.txt" + { + cat ${tagname}.WHATSNEW.txt + echo "" + echo "================" + echo "Download Details" + echo "================" + echo "" + echo "The uncompressed tarballs and patch files have been signed" + echo "using GnuPG (ID ${GPG_KEYID}). The source code can be downloaded" + echo "from:" + echo "" + echo " ${download_url}" + echo "" + echo "The release notes are available online at:" + echo "" + echo " ${download_url}${tagname}.WHATSNEW.txt" + echo "" + echo "Our Code, Our Bugs, Our Responsibility." + echo "(https://bugzilla.samba.org/)" + echo "" + echo " --Enjoy" + echo " The Samba Team" + } >announce.${tagname}.mail.txt + + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.mutt-arguments.txt" + { + echo -n "-i announce.${tagname}.mail.txt " + echo -n "-s \"$(cat announce.${tagname}.subject.txt | xargs)\" " + echo -n "$(cat announce.${tagname}.to.txt | xargs)" + } >announce.${tagname}.mutt-arguments.txt + + local headlinefile="posted_news/@UTCTIME@.${version}.headline.html" + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.headline.html" + { + echo "<!-- BEGIN: ${headlinefile} -->" + echo "<li> @UTCDATE@ <a href=\"${href}\">Samba ${version} Available for Download</a></li>" + echo "<!-- END: ${headlinefile} -->" + } >announce.${tagname}.headline.html + + local bodyfile="posted_news/@UTCTIME@.${version}.body.html" + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.body.html" + { + echo "<!-- BEGIN: ${bodyfile} -->" + echo "<h5><a name=\"${version}\">@UTCDATE@</a></h5>" + echo "<p class="headline">Samba ${version} Available for Download</p>" + echo "<p>" + echo "This is the ${rcname} release candidate of the upcoming Samba ${series} release series." + echo "</p>" + echo "<p>" + echo "The uncompressed tarball has been signed using GnuPG (ID ${GPG_KEYID})." + echo "The source code can be <a href=\"${download_url}${tagname}.tar.gz\">downloaded now</a>." + echo "See <a href=\"${download_url}${tagname}.WHATSNEW.txt\">the release notes for more info</a>." + echo "</p>" + echo "<!-- END: ${bodyfile} -->" + } >announce.${tagname}.body.html + + local webrepo="${TMPDIR}/webrepo" + + mkdir "${webrepo}" || { + return 1 + } + git -C "${webrepo}" init || { + return 1 + } + + mkdir -p "$(dirname ${webrepo}/${headlinefile})" || { + return 1 + } + cp -a "announce.${tagname}.headline.html" "${webrepo}/${headlinefile}" || { + return 1 + } + + mkdir -p "$(dirname ${webrepo}/${bodyfile})" || { + return 1 + } + cp -a "announce.${tagname}.body.html" "${webrepo}/${bodyfile}" || { + return 1 + } + + git -C "${webrepo}" add "${headlinefile}" "${bodyfile}" || { + return 1 + } + git -C "${webrepo}" commit --signoff --message "NEWS[${version}]: Samba ${version} Available for Download" || { + return 1 + } + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.patch.txt" + git -C "${webrepo}" format-patch --stdout -1 HEAD >announce.${tagname}.patch.txt || { + return 1 + } + + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.todo.txt" + { + ls -lart announce.${tagname}.* + echo "" + echo "NOTICE:" + echo "You need to do the following manual steps in order" + echo "to finish the announcement of ${tagname}!" + echo "" + echo "Change to a samba-web checkout and run" + echo " ./announce_samba_release.sh ${version} $(pwd)/announce.${tagname}.patch.txt" + echo "" + echo "Once the resulting commit is pushed a cron job will update " + echo "the content exported by the webserver every 5-10 mins." + echo "Check https://www.samba.org" + echo "" + echo "If the web content is updated, you need to send the announce mail (gpg signed)." + echo "- announce.${tagname}.to.txt contains the mail's recipients for the To: header." + echo "- announce.${tagname}.subject.txt contains the mail's subject line." + echo "- announce.${tagname}.mail.txt contains the content of the mail body." + echo "In case your're using mutt, you can use the following shortcut:" + echo " eval mutt \$(cat announce.${tagname}.mutt-arguments.txt)" + echo "" + echo "NOTICE: you're not done yet! Read the above instructions carefully!" + echo "See: announce.${tagname}.todo.txt" + echo "" + } >announce.${tagname}.todo.txt + + ls -lart announce.${tagname}.* + return 0 +} + +announcement_samba_stable() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + require_tagname "${FUNCNAME}" + + load_samba_stable_versions + + test -f "${tagname}.tar.gz" || { + echo "${tagname}.tar.gz does not exist" + return 1 + } + + local release_url="${download_url}samba/stable/" + local patch_url="${download_url}samba/patches/" + + echo "extract WHATSNEW.txt" + tar xf ${tagname}.tar.gz --to-stdout ${tagname}/WHATSNEW.txt >${TMPDIR}/WHATSNEW.txt + + local t="" + local oldversion=$(echo "${oldtagname}" | sed -e 's!^samba-!!') + local version=$(echo "${tagname}" | sed -e 's!^samba-!!') + local href="#${version}" + local series=$(echo "${version}" | cut -d '.' -f1-2) + local release=$(echo "${version}" | cut -d '.' -f3) + local releasename="latest" + case "${release}" in + 1) + releasename="first" + ;; + *) + releasename="latest" + ;; + esac + + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.to.txt" + { + echo "samba-announce@lists.samba.org, samba@lists.samba.org, samba-technical@lists.samba.org" + } >announce.${tagname}.to.txt + + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.subject.txt" + { + echo "[Announce] Samba ${version} Available for Download" + } >announce.${tagname}.subject.txt + + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.mail.txt" + { + local top=$(cat ${TMPDIR}/WHATSNEW.txt | grep -n '^Release notes for older releases follow:' | head -1 | cut -d ':' -f1) + test -n "${top}" || { + top=$(cat ${TMPDIR}/WHATSNEW.txt | wc -l) + } + local skip=$(cat ${TMPDIR}/WHATSNEW.txt | grep -n '^[^ ]' | head -1 | cut -d ':' -f1) + local headlimit=$(expr ${top} - 1) + local taillimit=$(expr ${headlimit} - \( ${skip} - 1 \)) + + echo "" + echo "" + echo "Release Announcements" + echo "---------------------" + echo "" + head -${headlimit} ${TMPDIR}/WHATSNEW.txt | tail -${taillimit} + echo "" + echo "================" + echo "Download Details" + echo "================" + echo "" + echo "The uncompressed tarballs and patch files have been signed" + echo "using GnuPG (ID ${GPG_KEYID}). The source code can be downloaded" + echo "from:" + echo "" + echo " ${release_url}" + echo "" + echo "The release notes are available online at:" + echo "" + echo " ${history_url}${tagname}.html" + echo "" + echo "Our Code, Our Bugs, Our Responsibility." + echo "(https://bugzilla.samba.org/)" + echo "" + echo " --Enjoy" + echo " The Samba Team" + } >announce.${tagname}.mail.txt + + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.mutt-arguments.txt" + { + echo -n "-i announce.${tagname}.mail.txt " + echo -n "-s \"$(cat announce.${tagname}.subject.txt | xargs)\" " + echo -n "$(cat announce.${tagname}.to.txt | xargs)" + } >announce.${tagname}.mutt-arguments.txt + + local htmlfile="history/${tagname}.html" + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.html" + { + local tmp=$(cat ${TMPDIR}/WHATSNEW.txt | grep -n '^Reporting bugs & Development Discussion' | head -1 | cut -d ':' -f1) + local lines=$(expr ${tmp} - 2) + + echo '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"' + echo ' "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">' + echo '<html xmlns="http://www.w3.org/1999/xhtml">' + + echo "<head>" + echo "<title>Samba ${version} - Release Notes</title>" + echo "</head>" + + echo "<body>" + echo "<H2>Samba ${version} Available for Download</H2>" + + echo "<p>" + echo "<a href=\"${release_url}${tagname}.tar.gz\">Samba ${version} (gzipped)</a><br>" + echo "<a href=\"${release_url}${tagname}.tar.asc\">Signature</a>" + echo "</p>" + + test -n "${patchfile}" && { + echo "<p>" + echo "<a href=\"${patch_url}${patchfile}.gz\">Patch (gzipped) against Samba ${oldversion}</a><br>" + echo "<a href=\"${patch_url}${patchfile}.asc\">Signature</a>" + echo "</p>" + } + + echo "<p>" + echo "<pre>" + head -${lines} ${TMPDIR}/WHATSNEW.txt | sed \ + -e 's!&!\&!g' | sed \ + -e 's!<!\<!g' \ + -e 's!>!\>!g' \ + -e 's!ä!\ä!g' \ + -e 's!Ä!\Ä!g' \ + -e 's!ö!\ö!g' \ + -e 's!Ö!\Ö!g' \ + -e 's!ü!\ü!g' \ + -e 's!Ãœ!\Ü!g' \ + -e 's!ß!\ß!g' \ + -e 's!"!\"!g' \ + -e "s!'!\'!g" | + cat + echo "</pre>" + echo "</p>" + + echo "</body>" + echo "</html>" + } >announce.${tagname}.html + + local headlinefile="posted_news/@UTCTIME@.${version}.headline.html" + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.headline.html" + { + echo "<!-- BEGIN: ${headlinefile} -->" + echo "<li> @UTCDATE@ <a href=\"${href}\">Samba ${version} Available for Download</a></li>" + echo "<!-- END: ${headlinefile} -->" + } >announce.${tagname}.headline.html + + local bodyfile="posted_news/@UTCTIME@.${version}.body.html" + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.body.html" + { + echo "<!-- BEGIN: ${bodyfile} -->" + echo "<h5><a name=\"${version}\">@UTCDATE@</a></h5>" + echo "<p class="headline">Samba ${version} Available for Download</p>" + echo "<p>" + echo "This is the ${releasename} stable release of the Samba ${series} release series." + echo "</p>" + echo "<p>" + echo "The uncompressed tarball has been signed using GnuPG (ID ${GPG_KEYID})." + echo "The source code can be <a href=\"${release_url}${tagname}.tar.gz\">downloaded now</a>." + test -n "${patchfile}" && { + echo "A <a href=\"${patch_url}${patchfile}.gz\">patch against Samba ${oldversion}</a> is also available." + } + echo "See <a href=\"${history_url}${tagname}.html\">the release notes for more info</a>." + echo "</p>" + echo "<!-- END: ${bodyfile} -->" + } >announce.${tagname}.body.html + + local webrepo="${TMPDIR}/webrepo" + + mkdir "${webrepo}" || { + return 1 + } + git -C "${webrepo}" init || { + return 1 + } + + mkdir -p "$(dirname ${webrepo}/${htmlfile})" || { + return 1 + } + cp -a "announce.${tagname}.html" "${webrepo}/${htmlfile}" || { + return 1 + } + + mkdir -p "$(dirname ${webrepo}/${headlinefile})" || { + return 1 + } + cp -a "announce.${tagname}.headline.html" "${webrepo}/${headlinefile}" || { + return 1 + } + + mkdir -p "$(dirname ${webrepo}/${bodyfile})" || { + return 1 + } + cp -a "announce.${tagname}.body.html" "${webrepo}/${bodyfile}" || { + return 1 + } + + git -C "${webrepo}" add "${htmlfile}" "${headlinefile}" "${bodyfile}" || { + return 1 + } + git -C "${webrepo}" commit --signoff --message "NEWS[${version}]: Samba ${version} Available for Download" || { + return 1 + } + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.patch.txt" + git -C "${webrepo}" format-patch --stdout -1 HEAD >announce.${tagname}.patch.txt || { + return 1 + } + + CLEANUP_FILES="${CLEANUP_FILES} announce.${tagname}.todo.txt" + { + ls -lart announce.${tagname}.* + echo "" + echo "NOTICE:" + echo "You need to do the following manual steps in order" + echo "to finish the announcement of ${tagname}!" + echo "" + echo "Change to a samba-web checkout and run" + echo " ./announce_samba_release.sh ${version} $(pwd)/announce.${tagname}.patch.txt" + echo "" + echo "Once the resulting commit is pushed a cron job will update " + echo "the content exported by the webserver every 5-10 mins." + echo "Check https://www.samba.org" + echo "" + echo "If the web content is updated, you need to send the announce mail (gpg signed)." + echo "- announce.${tagname}.to.txt contains the mail's recipients for the To: header." + echo "- announce.${tagname}.subject.txt contains the mail's subject line." + echo "- announce.${tagname}.mail.txt contains the content of the mail body." + echo "In case your're using mutt, you can use the following shortcut:" + echo " eval mutt \$(cat announce.${tagname}.mutt-arguments.txt)" + echo "" + echo "NOTICE: you're not done yet! Read the above instructions carefully!" + echo "See: announce.${tagname}.todo.txt" + echo "" + } >announce.${tagname}.todo.txt + + ls -lart announce.${tagname}.* + return 0 +} + +announcement_release() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + + test -n "${announcement_fn}" || { + echo "announcement_fn variable empty" + return 1 + } + + echo "Running ${announcement_fn}" + ${announcement_fn} +} + +announce_release() +{ + check_args "${FUNCNAME}" "$#" "0" || return 1 + require_tagname "${FUNCNAME}" + + test -f "announce.${tagname}.todo.txt" || { + echo "announce.${tagname}.todo.txt does not exist" + return 1 + } + + cat announce.${tagname}.todo.txt + return 0 +} + +case "${product}" in +talloc | tdb | tevent | ldb) + test -z "${GPG_USER-}" && { + GPG_USER='Samba Library Distribution Key <samba-bugs@samba.org>' + } + + test -z "${GPG_KEYID-}" && { + GPG_KEYID='4793916113084025' + } + + productbase="${product}" + srcdir="lib/${product}" + repo_url="${CONF_REPO_URL}" + upload_url="${CONF_UPLOAD_URL}/${product}/" + download_url="${CONF_DOWNLOAD_URL}/${product}/" + + check_fn="check_nopatch" + upload_fn="upload_nopatch" + fullcmds="create check push upload" + ;; +samba-rc) + test -z "${GPG_USER-}" && { + GPG_USER='Samba Distribution Verification Key <samba-bugs@samba.org>' + } + + test -z "${GPG_KEYID-}" && { + GPG_KEYID='AA99442FB680B620' + } + + productbase="samba" + srcdir="." + repo_url="${CONF_REPO_URL}" + upload_url="${CONF_UPLOAD_URL}/samba/rc/" + download_url="${CONF_DOWNLOAD_URL}/samba/rc/" + + verify_fn="verify_samba_rc" + check_fn="check_nopatch" + upload_fn="upload_nopatch" + announcement_fn="announcement_samba_rc" + fullcmds="verify create check whatsnew announcement push upload announce" + ;; +samba-stable) + test -z "${GPG_USER-}" && { + GPG_USER='Samba Distribution Verification Key <samba-bugs@samba.org>' + } + + test -z "${GPG_KEYID-}" && { + GPG_KEYID='AA99442FB680B620' + } + + productbase="samba" + srcdir="." + repo_url="${CONF_REPO_URL}" + upload_url="${CONF_UPLOAD_URL}/" + download_url="${CONF_DOWNLOAD_URL}/" + history_url="${CONF_HISTORY_URL}/samba/history/" + + verify_fn="verify_samba_stable" + check_fn="check_samba_stable" + upload_fn="upload_samba_stable" + announcement_fn="announcement_samba_stable" + fullcmds="verify create patch check announcement push upload announce" + ;; +TODO-samba-security) + test -z "${GPG_USER-}" && { + GPG_USER='Samba Distribution Verification Key <samba-bugs@samba.org>' + } + + test -z "${GPG_KEYID-}" && { + GPG_KEYID='AA99442FB680B620' + } + + productbase="samba" + srcdir="." + repo_url="${CONF_REPO_URL}" + upload_url="${CONF_UPLOAD_URL}/" + download_url="${CONF_DOWNLOAD_URL}/" + history_url="${CONF_HISTORY_URL}/samba/history/" + + verify_fn="verify_samba_stable" + check_fn="check_samba_stable" + upload_fn="upload_samba_stable" + announcement_fn="announcement_samba_security" + fullcmds="verify create patch check announcement" + next_cmd="push" + ;; +*) + usage + echo "Unknown product ${product}" + exit 1 + ;; +esac + +pushd ${srcdir} || { + echo "srcdir[${srcdir}] does not exist" + exit 1 +} + +trap_handler() +{ + echo "" + echo "ERROR: cleaning up" + echo "" + + for t in ${CLEANUP_TAGS}; do + echo "Removing tag[${t}]" + git tag -v "${t}" && { + git tag -d "${t}" || { + echo "failed to remove tag ${t}" + } + } + done + + for f in ${CLEANUP_FILES}; do + echo "Removing file[${f}]" + test -f "${f}" && { + rm "${f}" || { + echo "failed to remove ${f}" + } + } + done + + for d in ${CLEANUP_DIRS}; do + echo "Removing dir[${d}]" + test -d "${d}" && { + rm -rf "${d}" || { + echo "failed to remove ${d}" + } + } + done +} + +CLEANUP_TAGS="" +CLEANUP_FILES="" +CLEANUP_DIRS="" +trap trap_handler INT QUIT TERM EXIT + +cmd_allowed "${globalcmd}" fullrelease ${fullcmds} || { + usage + echo "command[${globalcmd}] not supported for product[${product}]" + exit 1 +} + +case "${globalcmd}" in +fullrelease) + check_args "${globalcmd}" "$#" "0" || exit 1 + cmds="${fullcmds}" + ;; +create) + check_args "${globalcmd}" "$#" "0" || exit 1 + check_args "create" "$#" "0" || exit 1 + + cmds="" + cmd_allowed "verify" ${fullcmds} && { + cmds="${cmds} verify" + } + cmds="${cmds} create" + cmd_allowed "whatsnew" ${fullcmds} && { + cmds="${cmds} whatsnew" + } + cmd_allowed "patch" ${fullcmds} && { + cmds="${cmds} patch" + } + cmds="${cmds} check" + cmd_allowed "announcement" ${fullcmds} && { + cmds="${cmds} announcement" + } + next_cmd="push" + ;; +push) + check_args "${globalcmd}" "$#" "1" || exit 1 + tagname="$1" + cmds="check push" + next_cmd="upload" + ;; +upload) + check_args "${globalcmd}" "$#" "1" || exit 1 + tagname="$1" + cmds="check upload" + cmd_allowed "announce" ${fullcmds} && { + next_cmd="announce" + } + ;; +announce) + check_args "${globalcmd}" "$#" "1" || exit 1 + tagname="$1" + cmds="check announce" + ;; +*) + usage + echo "Unknown command ${globalcmd}" + exit 1 + ;; +esac + +TMPDIR="release.$$" +CLEANUP_DIRS="${CLEANUP_DIRS} ${TMPDIR}" +umask 0077 +mkdir "${TMPDIR}" +umask 0022 + +for cmd in ${cmds}; do + echo "Starting subcommand[${cmd}]" + ${cmd}_release || { + echo "Failed subcommand[${cmd}]" + exit 1 + } + echo "Finished subcommand[${cmd}]" +done + +test -d "${TMPDIR}" && { + rm -rf "${TMPDIR}" || { + echo "failed to remove ${TMPDIR}" + } +} + +test -n "${next_cmd}" && { + echo "Continue with '$0 ${product} ${next_cmd} ${tagname}'." +} + +trap - INT QUIT TERM EXIT + +exit 0 diff --git a/script/show_test_time b/script/show_test_time new file mode 100755 index 0000000..70d29d7 --- /dev/null +++ b/script/show_test_time @@ -0,0 +1,29 @@ +#!/usr/bin/python +import optparse +import os.path +import subprocess +import sys + +parser = optparse.OptionParser() +parser.add_option("--limit", dest="limit", type=int, + help="Limit to this number of output entries.", default=0) +(opts, args) = parser.parse_args() + +durations = {} + +cmd = "subunit-1to2 | subunit-ls --times --no-passthrough" + +p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=sys.stdin, shell=True) +for l in p.stdout: + l = l.strip() + (name, duration) = l.rsplit(" ", 1) + durations[name] = float(duration) + +if opts.limit: + print("Top %d tests by run time:" % opts.limit) + +for i, (name, length) in enumerate(sorted( + durations.items(), key=lambda x: x[1], reverse=True)): + if opts.limit and i == opts.limit: + break + print("%d: %s -> %ds" % (i+1, name, length)) diff --git a/script/show_testsuite_time b/script/show_testsuite_time new file mode 100755 index 0000000..6e5808a --- /dev/null +++ b/script/show_testsuite_time @@ -0,0 +1,51 @@ +#!/usr/bin/env perl +use Time::Local ('timegm'); +my $in = STDIN; +use strict; + +my $intest=0; +my $name; +my $start=0; +my $end=0; +my %hash; +my $fh; +my $max=0; +if ($#ARGV >= 0) { + open($fh, "<", $ARGV[0]) || die "can't open ".$ARGV[0]; +} else { + $fh = $in; +} +if ($#ARGV >= 1) { + $max = $ARGV[1]; + if ($max =~ /\D/) { + die "not a decimal number: '$max'"; + } +} + +print "TOP $max slowest tests\n"; + +while(<$fh>) +{ + if (m/^testsuite: (.*)/) { + $intest = 1; + $name = $1; + } + if (m/testsuite-\w+:/) { + $hash{"$name -> ".($end - $start)} = $end - $start; + $intest = 0; + $start = 0; + } + if (m/^time: (\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)/ && $intest) { + my $ts=timegm($6,$5,$4,$3,$2 - 1,$1 - 1900); + if ($start == 0) { + $start = $ts; + } else { + $end = $ts; + } + } +} +my @sorted = sort { $hash{$b}<=>$hash{$a} } keys(%hash); +$max = @sorted if (($max <= 0) or ($max > @sorted)); +for my $l (@sorted[0..($max - 1)]) { + print $l."\n"; +} diff --git a/script/traffic_learner b/script/traffic_learner new file mode 100755 index 0000000..303956e --- /dev/null +++ b/script/traffic_learner @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# Generate a traffic model from a traffic summary file +# +# Copyright (C) Catalyst IT Ltd. 2017 +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +import sys +import argparse + +sys.path.insert(0, "bin/python") +from samba.emulate import traffic + +from samba.logger import get_samba_logger +logger = get_samba_logger(name=__name__, level=20) +error = logger.error +info = logger.info + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('-o', '--out', + help="write model here") + parser.add_argument('--dns-mode', choices=['inline', 'count'], + help='how to deal with DNS', default='count') + parser.add_argument('SUMMARY_FILE', nargs='*', type=argparse.FileType('r'), + default=[sys.stdin], + help="read from this file (default STDIN)") + args = parser.parse_args() + + if args.out is None: + error("No output file was specified to write the model to.") + error("Please specify a filename using the --out option.") + return 1 + + try: + outfile = open(args.out, 'w') + except IOError as e: + error("could not open %s" % args.out) + error(e) + return 1 + + if args.SUMMARY_FILE is sys.stdin: + info("reading from STDIN...") + + (conversations, + interval, + duration, + dns_counts) = traffic.ingest_summaries(args.SUMMARY_FILE, + dns_mode=args.dns_mode) + + model = traffic.TrafficModel() + info("learning model") + if args.dns_mode == 'count': + model.learn(conversations, dns_counts) + else: + model.learn(conversations) + + model.save(args.out) + +sys.exit(main()) diff --git a/script/traffic_replay b/script/traffic_replay new file mode 100755 index 0000000..60b7adb --- /dev/null +++ b/script/traffic_replay @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +# Generates samba network traffic +# +# Copyright (C) Catalyst IT Ltd. 2017 +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +import sys +import os +import optparse +import tempfile +import shutil +import random + +sys.path.insert(0, "bin/python") + +from samba import gensec, get_debug_level +from samba.emulate import traffic +import samba.getopt as options +from samba.logger import get_samba_logger +from samba.samdb import SamDB +from samba.auth import system_session + + +def print_err(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def main(): + + desc = ("Generates network traffic 'conversations' based on a model generated" + " by script/traffic_learner. This traffic is sent to <dns-hostname>," + " which is the full DNS hostname of the DC being tested.") + + parser = optparse.OptionParser( + "%prog [--help|options] <model-file> <dns-hostname>", + description=desc) + + parser.add_option('--dns-rate', type='float', default=0, + help='fire extra DNS packets at this rate') + parser.add_option('--dns-query-file', dest="dns_query_file", + help='A file contains DNS query list') + parser.add_option('-B', '--badpassword-frequency', + type='float', default=0.0, + help='frequency of connections with bad passwords') + parser.add_option('-K', '--prefer-kerberos', + action="store_true", + help='prefer kerberos when authenticating test users') + parser.add_option('-I', '--instance-id', type='int', default=0, + help='Instance number, when running multiple instances') + parser.add_option('-t', '--timing-data', + help=('write individual message timing data here ' + '(- for stdout)')) + parser.add_option('--preserve-tempdir', default=False, action="store_true", + help='do not delete temporary files') + parser.add_option('-F', '--fixed-password', + type='string', default=None, + help=('Password used for the test users created. ' + 'Required')) + parser.add_option('-c', '--clean-up', + action="store_true", + help='Clean up the generated groups and user accounts') + parser.add_option('--random-seed', type='int', default=None, + help='Use to keep randomness consistent across multiple runs') + parser.add_option('--stop-on-any-error', + action="store_true", + help='abort the whole thing if a child fails') + model_group = optparse.OptionGroup(parser, 'Traffic Model Options', + 'These options alter the traffic ' + 'generated by the model') + model_group.add_option('-S', '--scale-traffic', type='float', + help=('Increase the number of conversations by ' + 'this factor (or use -T)')) + parser.add_option('-T', '--packets-per-second', type=float, + help=('attempt this many packets per second ' + '(alternative to -S)')) + parser.add_option('--old-scale', + action="store_true", + help='emulate the old scale for traffic') + model_group.add_option('-D', '--duration', type='float', default=60.0, + help=('Run model for this long (approx). ' + 'Default 60s for models')) + model_group.add_option('--latency-timeout', type='float', default=None, + help=('Wait this long for last packet to finish')) + model_group.add_option('-r', '--replay-rate', type='float', default=1.0, + help='Replay the traffic faster by this factor') + model_group.add_option('--conversation-persistence', type='float', + default=0.0, + help=('chance (0 to 1) that a conversation waits ' + 'when it would have died')) + model_group.add_option('--traffic-summary', + help=('Generate a traffic summary file and write ' + 'it here (- for stdout)')) + parser.add_option_group(model_group) + + user_gen_group = optparse.OptionGroup(parser, 'Generate User Options', + "Add extra user/groups on the DC to " + "increase the DB size. These extra " + "users aren't used for traffic " + "generation.") + user_gen_group.add_option('-G', '--generate-users-only', + action="store_true", + help='Generate the users, but do not replay ' + 'the traffic') + user_gen_group.add_option('-n', '--number-of-users', type='int', default=0, + help='Total number of test users to create') + user_gen_group.add_option('--number-of-groups', type='int', default=None, + help='Create this many groups') + user_gen_group.add_option('--average-groups-per-user', + type='int', default=0, + help='Assign the test users to this ' + 'many groups on average') + user_gen_group.add_option('--group-memberships', type='int', default=0, + help='Total memberships to assign across all ' + 'test users and all groups') + user_gen_group.add_option('--max-members', type='int', default=None, + help='Max users to add to any one group') + parser.add_option_group(user_gen_group) + + sambaopts = options.SambaOptions(parser) + parser.add_option_group(sambaopts) + parser.add_option_group(options.VersionOptions(parser)) + credopts = options.CredentialsOptions(parser) + parser.add_option_group(credopts) + + # the --no-password credential doesn't make sense for this tool + if parser.has_option('-N'): + parser.remove_option('-N') + + opts, args = parser.parse_args() + + # First ensure we have reasonable arguments + + if len(args) == 1: + model_file = None + host = args[0] + elif len(args) == 2: + model_file, host = args + else: + parser.print_usage() + return + + lp = sambaopts.get_loadparm() + debuglevel = get_debug_level() + logger = get_samba_logger(name=__name__, + verbose=debuglevel > 3, + quiet=debuglevel < 1) + + traffic.DEBUG_LEVEL = debuglevel + # pass log level down to traffic module to make sure level is controlled + traffic.LOGGER.setLevel(logger.getEffectiveLevel()) + + if opts.clean_up: + logger.info("Removing user and machine accounts") + lp = sambaopts.get_loadparm() + creds = credopts.get_credentials(lp) + creds.set_gensec_features(creds.get_gensec_features() | gensec.FEATURE_SEAL) + ldb = traffic.openLdb(host, creds, lp) + traffic.clean_up_accounts(ldb, opts.instance_id) + exit(0) + + if model_file: + if not os.path.exists(model_file): + logger.error("Model file %s doesn't exist" % model_file) + sys.exit(1) + # the model-file can be omitted for --generate-users-only and + # --cleanup-up, but it should be specified in all other cases + elif not opts.generate_users_only: + logger.error("No model file specified to replay traffic from") + sys.exit(1) + + if not opts.fixed_password: + logger.error(("Please use --fixed-password to specify a password" + " for the users created as part of this test")) + sys.exit(1) + + if opts.random_seed is not None: + random.seed(opts.random_seed) + + creds = credopts.get_credentials(lp) + creds.set_gensec_features(creds.get_gensec_features() | gensec.FEATURE_SEAL) + + domain = creds.get_domain() + if domain: + lp.set("workgroup", domain) + else: + domain = lp.get("workgroup") + if domain == "WORKGROUP": + logger.error(("NETBIOS domain does not appear to be " + "specified, use the --workgroup option")) + sys.exit(1) + + if not opts.realm and not lp.get('realm'): + logger.error("Realm not specified, use the --realm option") + sys.exit(1) + + if opts.generate_users_only and not (opts.number_of_users or + opts.number_of_groups): + logger.error(("Please specify the number of users and/or groups " + "to generate.")) + sys.exit(1) + + if opts.group_memberships and opts.average_groups_per_user: + logger.error(("--group-memberships and --average-groups-per-user" + " are incompatible options - use one or the other")) + sys.exit(1) + + if not opts.number_of_groups and opts.average_groups_per_user: + logger.error(("--average-groups-per-user requires " + "--number-of-groups")) + sys.exit(1) + + if opts.number_of_groups and opts.average_groups_per_user: + if opts.number_of_groups < opts.average_groups_per_user: + logger.error(("--average-groups-per-user can not be more than " + "--number-of-groups")) + sys.exit(1) + + if not opts.number_of_groups and opts.group_memberships: + logger.error("--group-memberships requires --number-of-groups") + sys.exit(1) + + if opts.scale_traffic is not None and opts.packets_per_second is not None: + logger.error("--scale-traffic and --packets-per-second " + "are incompatible. Use one or the other.") + sys.exit(1) + + if not opts.scale_traffic and not opts.packets_per_second: + logger.info("No packet rate specified. Using --scale-traffic=1.0") + opts.scale_traffic = 1.0 + + if opts.timing_data not in ('-', None): + try: + open(opts.timing_data, 'w').close() + except IOError: + # exception info will be added to log automatically + logger.exception(("the supplied timing data destination " + "(%s) is not writable" % opts.timing_data)) + sys.exit() + + if opts.traffic_summary not in ('-', None): + try: + open(opts.traffic_summary, 'w').close() + except IOError: + # exception info will be added to log automatically + if debuglevel > 0: + import traceback + traceback.print_exc() + logger.exception(("the supplied traffic summary destination " + "(%s) is not writable" % opts.traffic_summary)) + sys.exit() + + if opts.old_scale: + # we used to use a silly calculation based on the number + # of conversations; now we use the number of packets and + # scale traffic accurately. To roughly compare with older + # numbers you use --old-scale which approximates as follows: + opts.scale_traffic *= 0.55 + + # ingest the model + if model_file and not opts.generate_users_only: + model = traffic.TrafficModel() + try: + model.load(model_file) + except ValueError: + if debuglevel > 0: + import traceback + traceback.print_exc() + logger.error(("Could not parse %s, which does not seem to be " + "a model generated by script/traffic_learner." + % model_file)) + sys.exit(1) + + logger.info(("Using the specified model file to " + "generate conversations")) + + if opts.scale_traffic: + packets_per_second = model.scale_to_packet_rate(opts.scale_traffic) + else: + packets_per_second = opts.packets_per_second + + conversations = \ + model.generate_conversation_sequences( + packets_per_second, + opts.duration, + opts.replay_rate, + opts.conversation_persistence) + else: + conversations = [] + + if opts.number_of_users and opts.number_of_users < len(conversations): + logger.error(("--number-of-users (%d) is less than the " + "number of conversations to replay (%d)" + % (opts.number_of_users, len(conversations)))) + sys.exit(1) + + number_of_users = max(opts.number_of_users, len(conversations)) + + if opts.number_of_groups is None: + opts.number_of_groups = max(int(number_of_users / 10), 1) + + max_memberships = number_of_users * opts.number_of_groups + + if not opts.group_memberships and opts.average_groups_per_user: + opts.group_memberships = opts.average_groups_per_user * number_of_users + logger.info(("Using %d group-memberships based on %u average " + "memberships for %d users" + % (opts.group_memberships, + opts.average_groups_per_user, number_of_users))) + + if opts.group_memberships > max_memberships: + logger.error(("The group memberships specified (%d) exceeds " + "the total users (%d) * total groups (%d)" + % (opts.group_memberships, number_of_users, + opts.number_of_groups))) + sys.exit(1) + + # if no groups were specified by the user, then make sure we create some + # group memberships (otherwise it's not really a fair test) + if not opts.group_memberships and not opts.average_groups_per_user: + opts.group_memberships = min(number_of_users * 5, max_memberships) + + # Get an LDB connection. + try: + # if we're only adding users, then it's OK to pass a sam.ldb filepath + # as the host, which creates the users much faster. In all other cases + # we should be connecting to a remote DC + if opts.generate_users_only and os.path.isfile(host): + ldb = SamDB(url="ldb://{0}".format(host), + session_info=system_session(), lp=lp) + else: + ldb = traffic.openLdb(host, creds, lp) + except: + logger.error(("\nInitial LDAP connection failed! Did you supply " + "a DNS host name and the correct credentials?")) + sys.exit(1) + + if opts.generate_users_only: + # generate computer accounts for added realism. Assume there will be + # some overhang with more computer accounts than users + computer_accounts = int(1.25 * number_of_users) + traffic.generate_users_and_groups(ldb, + opts.instance_id, + opts.fixed_password, + opts.number_of_users, + opts.number_of_groups, + opts.group_memberships, + opts.max_members, + machine_accounts=computer_accounts, + traffic_accounts=False) + sys.exit() + + tempdir = tempfile.mkdtemp(prefix="samba_tg_") + logger.info("Using temp dir %s" % tempdir) + + traffic.generate_users_and_groups(ldb, + opts.instance_id, + opts.fixed_password, + number_of_users, + opts.number_of_groups, + opts.group_memberships, + opts.max_members, + machine_accounts=len(conversations), + traffic_accounts=True) + + accounts = traffic.generate_replay_accounts(ldb, + opts.instance_id, + len(conversations), + opts.fixed_password) + + statsdir = traffic.mk_masked_dir(tempdir, 'stats') + + if opts.traffic_summary: + if opts.traffic_summary == '-': + summary_dest = sys.stdout + else: + summary_dest = open(opts.traffic_summary, 'w') + + logger.info("Writing traffic summary") + summaries = [] + for c in traffic.seq_to_conversations(conversations): + summaries += c.replay_as_summary_lines() + + summaries.sort() + for (time, line) in summaries: + print(line, file=summary_dest) + + exit(0) + + traffic.replay(conversations, + host, + lp=lp, + creds=creds, + accounts=accounts, + dns_rate=opts.dns_rate, + dns_query_file=opts.dns_query_file, + duration=opts.duration, + latency_timeout=opts.latency_timeout, + badpassword_frequency=opts.badpassword_frequency, + prefer_kerberos=opts.prefer_kerberos, + statsdir=statsdir, + domain=domain, + base_dn=ldb.domain_dn(), + ou=traffic.ou_name(ldb, opts.instance_id), + tempdir=tempdir, + stop_on_any_error=opts.stop_on_any_error, + domain_sid=ldb.get_domain_sid(), + instance_id=opts.instance_id) + + if opts.timing_data == '-': + timing_dest = sys.stdout + elif opts.timing_data is None: + timing_dest = None + else: + timing_dest = open(opts.timing_data, 'w') + + logger.info("Generating statistics") + traffic.generate_stats(statsdir, timing_dest) + + if not opts.preserve_tempdir: + logger.info("Removing temporary directory") + shutil.rmtree(tempdir) + else: + # delete the empty directories anyway. There are thousands of + # them and they're EMPTY. + for d in os.listdir(tempdir): + if d.startswith('conversation-'): + path = os.path.join(tempdir, d) + try: + os.rmdir(path) + except OSError as e: + logger.info("not removing %s (%s)" % (path, e)) + +main() diff --git a/script/traffic_summary.pl b/script/traffic_summary.pl new file mode 100755 index 0000000..295a320 --- /dev/null +++ b/script/traffic_summary.pl @@ -0,0 +1,707 @@ +#! /usr/bin/perl +# +# Summarise tshark pdml output into a form suitable for the load test tool +# +# Copyright (C) Catalyst.Net Ltd 2017 +# +# Catalyst.Net's contribution was written by Gary Lockyer +# <gary@catalyst.net.nz>. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +use warnings; +use strict; + +use Getopt::Long; +use Pod::Usage; + +BEGIN { + unless (eval "require XML::Twig") { + warn "traffic_summary requires the perl module XML::Twig\n" . + "on Ubuntu/Debian releases run\n". + " sudo apt install libxml-twig-perl \n". + "or install from CPAN\n". + "\nThe reported error was:\n$@"; + exit(1); + } +} + + +my %ip_map; # Map of IP address to sequence number +my $ip_sequence = 0; # count of unique IP addresses seen + + +my $timestamp; # Packet timestamp +my $stream; # Wireshark stream number +my $ip_proto; # IP protocol (IANA protocol number) +my $source; # source IP address +my $dest; # destination address +my $proto; # application protocol name +my $description; # protocol specific description +my %proto_data; # protocol specific data captured for the current packet +my $malformed_packet; # Indicates the current packet has errors +my $ldap_filter; # cleaned ldap filter +my $ldap_attributes; # attributes requested in an ldap query + + + +# Dispatch table mapping the wireshark variables of interest to the +# functions responsible for processing them +my %field_dispatch_table = ( + 'timestamp' => \×tamp, + 'ip.src' => \&ip_src, + 'ipv6.src' => \&ip_src, + 'ip.dst' => \&ip_dst, + 'ipv6.dst' => \&ip_dst, + 'ip.proto' => \&ip_proto, + 'udp.stream' => \&stream, + 'tcp.stream' => \&stream, + 'dns.flags.opcode' => \&field_data, + 'dns.flags.response' => \&field_data, + 'netlogon.opnum' => \&field_data, + 'kerberos.msg_type' => \&field_data, + 'smb.cmd' => \&field_data, + 'smb2.cmd' => \&field_data, + 'ldap.protocolOp' => \&field_data, + 'gss-api.OID' => \&field_data, + 'ldap.gssapi_encrypted_payload' => \&field_data, + 'ldap.baseObject' => \&field_data, + 'ldap.scope' => \&field_data, + 'ldap.AttributeDescription' => \&ldap_attribute, + 'ldap.modification_element' => \&ldap_add_modify, + 'ldap.AttributeList_item_element' => \&ldap_add_modify, + 'ldap.operation' => \&field_data, + 'ldap.authentication' => \&field_data, + 'lsarpc.opnum' => \&field_data, + 'samr.opnum' => \&field_data, + 'dcerpc.pkt_type' => \&field_data, + 'epm.opnum' => \&field_data, + 'dnsserver.opnum' => \&field_data, + 'drsuapi.opnum' => \&field_data, + 'browser.command' => \&field_data, + 'smb_netlogon.command' => \&field_data, + 'srvsvc.opnum' => \&field_data, + 'nbns.flags.opcode' => \&field_data, + 'nbns.flags.response' => \&field_data, + '_ws.expert.message' => \&field_data, +); + +# Dispatch table mapping protocols to the routine responsible for formatting +# their output. Protocols not in this table are ignored. +# +my %proto_dispatch_table = ( + 'dns' => sub { return format_opcode( 'dns.flags.response')}, + 'rpc_netlogon' => sub { return format_opcode( 'netlogon.opnum')}, + 'kerberos' => \&format_kerberos, + 'smb' => sub { return format_opcode( 'smb.cmd')}, + 'smb2' => sub { return format_opcode( 'smb2.cmd')}, + 'ldap' => \&format_ldap, + 'cldap' => \&format_ldap, + 'lsarpc' => sub { return format_opcode( 'lsarpc.opnum')}, + 'samr' => sub { return format_opcode( 'samr.opnum')}, + 'dcerpc' => sub { return format_opcode( 'dcerpc.pkt_type')}, + 'epm' => sub { return format_opcode( 'epm.opnum')}, + 'dnsserver' => sub { return format_opcode( 'dnsserver.opnum')}, + 'drsuapi' => sub { return format_opcode( 'drsuapi.opnum')}, + 'browser' => sub { return format_opcode( 'browser.command')}, + 'smb_netlogon' => sub { return format_opcode( 'smb_netlogon.command')}, + 'srvsvc' => sub { return format_opcode( 'srvsvc.opnum')}, + 'nbns' => sub { return format_opcode( 'nbns.flags.response')}, +); + +# XPath entry to extract the kerberos cname +my $kerberos_cname_path = + 'packet/proto/field[@name = "kerberos.as_req_element"]' + . '/field[@name = "kerberos.req_body_element"]' + . '/field[@name = "kerberos.cname_element"]' + . '/field[@name = "kerberos.name_string"]' + . '/field[@name = "kerberos.KerberosString"]'; + +# XPath entry to extract the ldap filter +my $ldap_filter_path = + 'field[@name = "ldap.searchRequest_element"]/field'; + + +# Create an XML Twig parser and register the event handlers. +# +my $t = XML::Twig->new( + start_tag_handlers => { + 'packet' => \&packet_start, + }, + twig_handlers => { + 'packet' => \&packet, + 'proto' => \&protocol, + 'field' => \&field, + $kerberos_cname_path => \&kerberos_cname, + $ldap_filter_path => \&ldap_filter, + }, +); + +#------------------------------------------------------------------------------ +# Main loop +# +#------------------------------------------------------------------------------ +my $help = 0; +GetOptions( 'help|h' => \$help) or pod2usage(2); +pod2usage(1) if $help; + +if (@ARGV) { + foreach my $file (@ARGV) { + eval { + $t->parsefile( $file); + }; + if ($@) { + print STDERR "Unable to process $file, ". + "did you run tshark with the -T pdml option?"; + } + } +} else { + pod2usage(1) if -t STDIN; + eval { + $t->parse( \*STDIN); + }; + if ($@) { + print STDERR "Unable to process input, ". + "are you running tshark with the -T pdml option?"; + } +} + + +#------------------------------------------------------------------------------ +# New packet detected reset the globals +#------------------------------------------------------------------------------ +sub packet_start +{ + my ($t, $packet) = @_; + $timestamp = ""; + $stream = ""; + $ip_proto = ""; + $source = ""; + $dest = ""; + $description = undef; + %proto_data = (); + $malformed_packet = undef; + $ldap_filter = ""; + $ldap_attributes = ""; +} + +#------------------------------------------------------------------------------ +# Complete packet element parsed from the XML feed +# output the protocol summary if required +#------------------------------------------------------------------------------ +sub packet +{ + my ($t, $packet) = @_; + + my $data; + if (exists $proto_dispatch_table{$proto}) { + if ($malformed_packet) { + $data = "\t\t** Malformed Packet ** " . ($proto_data{'_ws.expert.message.show'} || ''); + } else { + my $rsub = $proto_dispatch_table{$proto}; + $data = &$rsub(); + } + print "$timestamp\t$ip_proto\t$stream\t$source\t$dest\t$proto\t$data\n"; + } + $t->purge; +} + +#------------------------------------------------------------------------------ +# Complete protocol element parsed from the XML input +# Update the protocol name +#------------------------------------------------------------------------------ +sub protocol +{ + my ($t, $protocol) = @_; + if ($protocol->{att}->{showname}) { + } + # Tag a packet as malformed if the protocol is _ws.malformed + # and the hide attribute is not 'yes' + if ($protocol->{att}->{name} eq '_ws.malformed' + && !($protocol->{att}->{hide} && $protocol->{att}->{hide} eq 'yes') + ) { + $malformed_packet = 1; + } + # Don't set the protocol name if it's a wireshark malformed + # protocol entry, or the packet was truncated during capture + my $p = $protocol->{att}->{name}; + if ($p ne '_ws.malformed' && $p ne '_ws.short') { + $proto = $p; + } +} + + +#------------------------------------------------------------------------------ +# Complete field element parsed, extract any data of interest +#------------------------------------------------------------------------------ +sub field +{ + my ($t, $field) = @_; + my $name = $field->{att}->{name}; + + # Only process the field if it has a corresponding entry in + # %field_dispatch_table + if (exists $field_dispatch_table{$name}) { + my $rsub = $field_dispatch_table{$name}; + &$rsub( $field); + } +} + +#------------------------------------------------------------------------------ +# Process a timestamp field element +#------------------------------------------------------------------------------ +sub timestamp +{ + my ($field) = @_; + $timestamp = $field->{att}->{value}; +} + +#------------------------------------------------------------------------------ +# Process a wireshark stream element, used to group a sequence of requests +# and responses between two IP addresses +#------------------------------------------------------------------------------ +sub stream +{ + my ($field) = @_; + $stream = $field->{att}->{show}; +} + +#------------------------------------------------------------------------------ +# Process a source ip address field, mapping the IP address to it's +# corresponding sequence number. +#------------------------------------------------------------------------------ +sub ip_src +{ + my ($field) = @_; + $source = map_ip( $field); +} + +#------------------------------------------------------------------------------ +# Process a destination ip address field, mapping the IP address to it's +# corresponding sequence number. +#------------------------------------------------------------------------------ +sub ip_dst +{ + my ($field) = @_; + $dest = map_ip( $field); +} + +#------------------------------------------------------------------------------ +# Process an ip protocol element, extracting IANA protocol number +#------------------------------------------------------------------------------ +sub ip_proto +{ + my ($field) = @_; + $ip_proto = $field->{att}->{value}; +} + + + +#------------------------------------------------------------------------------ +# Extract an ldap attribute and append it to ldap_attributes +#------------------------------------------------------------------------------ +sub ldap_attribute +{ + my ($field) = @_; + my $attribute = $field->{att}->{show}; + + if (defined $attribute) { + $ldap_attributes .= "," if $ldap_attributes; + $ldap_attributes .= $attribute; + } +} + +#------------------------------------------------------------------------------ +# Process a field element, extract the value, show and showname attributes +# and store them in the %proto_data hash. +# +#------------------------------------------------------------------------------ +sub field_data +{ + my ($field) = @_; + my $name = $field->{att}->{name}; + $proto_data{$name.'.value'} = $field->{att}->{value}; + $proto_data{$name.'.show'} = $field->{att}->{show}; + $proto_data{$name.'.showname'} = $field->{att}->{showname}; +} + +#------------------------------------------------------------------------------ +# Process a kerberos cname element, if the cname ends with a $ it's a machine +# name. Otherwise it's a user name. +# +#------------------------------------------------------------------------------ +sub kerberos_cname +{ + my ($t, $field) = @_; + my $cname = $field->{att}->{show}; + my $type; + if( $cname =~ /\$$/) { + $type = 'machine'; + } else { + $type = 'user'; + } + $proto_data{'kerberos.cname.type'} = $type; +} + + +#------------------------------------------------------------------------------ +# Process an ldap filter, remove the values but keep the attribute names +#------------------------------------------------------------------------------ +sub ldap_filter +{ + my ($t, $field) = @_; + if ( $field->{att}->{show} && $field->{att}->{show} =~ /^Filter:/) { + my $filter = $field->{att}->{show}; + + # extract and save the objectClass to keep the value + my @object_classes; + while ( $filter =~ m/\((objectClass=.*?)\)/g) { + push @object_classes, $1; + } + + # extract and save objectCategory and the top level value + my @object_categories; + while ( $filter =~ m/(\(objectCategory=.*?,|\(objectCategory=.*?\))/g + ) { + push @object_categories, $1; + } + + # Remove all the values from the attributes + # Input + # Filter: (nCName=DC=DomainDnsZones,DC=sub1,DC=ad,DC=rh,DC=at,DC=net) + # Output + # (nCName) + $filter =~ s/^Filter:\s*//; # Remove the 'Filter: ' prefix + $filter =~ s/=.*?\)/\)/g; # Remove from the = to the first ) + + # Now restore the parts of objectClass and objectCategory that are being + # retained + # + for my $cat (@object_categories) { + $filter =~ s/\(objectCategory\)/$cat/; + } + + for my $class (@object_classes) { + $filter =~ s/\(objectClass\)/($class)/; + } + + $ldap_filter = $filter; + } else { + # Ok not an ldap filter so call the default field handler + field( $t, $field); + } +} + + +#------------------------------------------------------------------------------ +# Extract the attributes from ldap modification and add requests +#------------------------------------------------------------------------------ +sub ldap_add_modify +{ + my ($field) = @_; + my $type = $field->first_child('field[@name="ldap.type"]'); + my $attribute = $type->{att}->{show} if $type; + if (defined $attribute) { + $ldap_attributes .= "," if $ldap_attributes; + $ldap_attributes .= $attribute; + } +} +#------------------------------------------------------------------------------ +# Map an IP address to a unique sequence number. Assigning it a sequence number +# if one has not already been assigned. +# +#------------------------------------------------------------------------------ +sub map_ip +{ + my ($field) = @_; + my $ip = $field->{att}->{show}; + if ( !exists( $ip_map{$ip})) { + $ip_sequence++; + $ip_map{$ip} = $ip_sequence; + } + return $ip_map{$ip}; +} + +#------------------------------------------------------------------------------ +# Format a protocol operation code for output. +# +#------------------------------------------------------------------------------ +sub format_opcode +{ + my ($name) = @_; + my $operation = $proto_data{$name.'.show'}; + my $description = $proto_data{$name.'.showname'} || ''; + + # Strip off the common prefix text, and the trailing (n). + # This tidies up most but not all descriptions. + $description =~ s/^[^:]*?: ?// if $description; + $description =~ s/^Message is a // if $description; + $description =~ s/\(\d+\)\s*$// if $description; + $description =~ s/\s*$// if $description; + + return "$operation\t$description"; +} + +#------------------------------------------------------------------------------ +# Format ldap protocol details for output +#------------------------------------------------------------------------------ +sub format_ldap +{ + my ($name) = @_; + if ( exists( $proto_data{'ldap.protocolOp.show'}) + || exists( $proto_data{'gss-api.OID.show'}) + ) { + my $operation = $proto_data{'ldap.protocolOp.show'}; + my $description = $proto_data{'ldap.protocolOp.showname'} || ''; + my $oid = $proto_data{'gss-api.OID.show'} || ''; + my $base_object = $proto_data{'ldap.baseObject.show'} || ''; + my $scope = $proto_data{'ldap.scope.show'} || ''; + + # Now extract operation specific data + my $extra; + my $extra_desc; + $operation = '' if !defined $operation; + if ($operation eq 6) { + # Modify operation + $extra = $proto_data{'ldap.operation.show'}; + $extra_desc = $proto_data{'ldap.operation.showname'}; + } elsif ($operation eq 0) { + # Bind operation + $extra = $proto_data{'ldap.authentication.show'}; + $extra_desc = $proto_data{'ldap.authentication.showname'}; + } + $extra = '' if !defined $extra; + $extra_desc = '' if !defined $extra_desc; + + + # strip the values out of the base object + if ($base_object) { + $base_object =~ s/^<//; # leading '<' if present + $base_object =~ s/>$//; # trailing '>' if present + $base_object =~ s/=.*?,/,/g; # from = up to the next comma + $base_object =~ s/=.*?$//; # from = up to the end of string + } + + # strip off the leading prefix on the extra_description + # and the trailing (n); + $extra_desc =~ s/^[^:]*?: ?// if $extra_desc; + $extra_desc =~ s/\(\d+\)\s*$// if $extra_desc; + $extra_desc =~ s/\s*$// if $extra_desc; + + # strip off the common prefix on the description + # and the trailing (n); + $description =~ s/^[^:]*?: ?// if $description; + $description =~ s/\(\d+\)\s*$// if $description; + $description =~ s/\s*$// if $description; + + return "$operation\t$description\t$scope\t$base_object" + ."\t$ldap_filter\t$ldap_attributes\t$extra\t$extra_desc\t$oid"; + } else { + return "\t*** Unknown ***"; + } +} + +#------------------------------------------------------------------------------ +# Format kerberos protocol details for output. +#------------------------------------------------------------------------------ +sub format_kerberos +{ + my $msg_type = $proto_data{'kerberos.msg_type.show'} || ''; + my $cname_type = $proto_data{'kerberos.cname.type'} || ''; + my $description = $proto_data{'kerberos.msg_type.showname'} || ''; + + # Tidy up the description + $description =~ s/^[^:]*?: ?// if $description; + $description =~ s/\(\d+\)\s*$// if $description; + $description =~ s/\s*$// if $description; + return "$msg_type\t$description\t$cname_type"; +} + +=pod + +=head1 NAME + +traffic_summary.pl - summarise tshark pdml output + +=head1 USAGE + +B<traffic_summary.pl> [FILE...] + +Summarise samba network traffic from tshark pdml output. Produces a tsv +delimited summary of samba activity. + +To process unencrypted traffic + + tshark -r capture.file -T pdml | traffic_summary.pl + +To process encrypted kerberos traffic + + tshark -r capture.file -K krb5.keytab -o kerberos.decrypt:true -T pdml | traffic_summary.pl + +To display more detailed documentation, including details of the output format + + perldoc traffic_summary.pl + + NOTE: tshark pdml output is very verbose, so it's better to pipe the tshark + output directly to traffic_summary, rather than generating + intermediate pdml format files. + +=head1 OPTIONS + B<--help> Display usage message and exit. + +=head1 DESCRIPTION + +Summarises tshark pdml output into a format suitable for load analysis +and input into load generation tools. + +It reads the pdml input from stdin or the list of files passed on the command line. + + +=head2 Output format + The output is tab delimited fields and one line per summarised packet. + +=head3 Fields + B<timestamp> Packet timestamp + B<IP protocol> The IANA protocol number + B<Wireshark Stream Number> Calculated by wireshark groups related requests and responses + B<Source IP> The unique sequence number for the source IP address + B<Destination IP> The unique sequence number for the destination IP address + B<protocol> The protocol name + B<opcode> The protocol operation code + B<Description> The protocol or operation description + B<extra> Extra protocol specific data, may be more than one field + + +=head2 IP address mapping + Rather than capturing and printing the IP addresses. Each unique IP address + seen is assigned a sequence number. So the first IP address seen will be 1, + the second 2 ... + +=head2 Packets collected + Packets containing the following protocol records are summarised: + dns + rpc_netlogon + kerberos + smb + smb2 + ldap + cldap + lsarpc + samr + dcerpc + epm + dnsserver + drsuapi + browser + smb_netlogon + srvsvc + nbns + + Any other packets are ignored. + + In addition to the standard elements extra data is returned for the following + protocol record. + +=head3 kerberos + cname_type machine cname ends with a $ + user cname does not end with a $ + +=head3 ldap + + scope Query Scope + 0 - Base + 1 - One level + 2 - sub tree + base_object ldap base object + ldap_filter the ldap filter, attribute names are retained but the values + are removed. + ldap_attributes ldap attributes, only the names are retained any values are + discarded, with the following two exceptions + objectClass all the attribute values are retained + objectCategory the top level value is retained + i.e. everything from the = to the first , + +=head3 ldap modifiyRequest + In addition to the standard ldap fields the modification type is also captured + + modify_operator for modifyRequests this contains the modify operation + 0 - add + 1 - delete + 2 - replace + modify_description a description of the operation if available + +=head3 modify bindRequest + In addition to the standard ldap fields details of the authentication + type are captured + + authentication type 0 - Simple + 3 - SASL + description Description of the authentication mechanism + oid GSS-API OID's + 1.2.840.113554.1.2.2 - Kerberos v5 + 1.2.840.48018.1.2.2 - Kerberos V5 + (incorrect, used by old Windows versions) + 1.3.6.1.5.5.2 - SPNEGO + 1.3.6.1.5.2.5 - IAKERB + 1.3.6.1.4.1.311.2.2.10 - NTLM SSP + 1.3.6.1.5.5.14 - SCRAM-SHA-1 + 1.3.6.1.5.5.18 - SCRAM-SHA-256 + 1.3.6.1.5.5.15.1.1.* - GSS-EAP + 1.3.6.1.5.2.7 - PKU2U + 1.3.6.1.5.5.1.1 - SPKM-1 + 1.3.6.1.5.5.1.2 - SPKM-2 + 1.3.6.1.5.5.1.3 - SPKM-3 + 1.3.6.1.5.5.9 - LIPKEY + 1.2.752.43.14.2 - NETLOGON + +=head1 DEPENDENCIES +tshark +XML::Twig For Ubuntu libxml-twig-perl, or from CPAN +use Getopt::Long +use Pod::Usage + + +=head1 Diagnostics + +=head2 ** Unknown ** +Unable to determine the operation being performed, for ldap it typically +indicates a kerberos encrypted operation. + +=head2 ** Malformed Packet ** +tshark indicated that the packet was malformed, for ldap it usually indicates TLS +encrypted traffic. + +=head1 LICENSE AND COPYRIGHT + + Copyright (C) Catalyst.Net Ltd 2017 + + Catalyst.Net's contribution was written by Gary Lockyer + <gary@catalyst.net.nz>. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + + +=cut |