diff options
Diffstat (limited to '')
| -rwxr-xr-x | pybuild | 602 | ||||
| -rwxr-xr-x | pybuild-autopkgtest | 68 | ||||
| -rw-r--r-- | pybuild-autopkgtest.rst | 109 | ||||
| -rw-r--r-- | pybuild.rst | 306 |
4 files changed, 1085 insertions, 0 deletions
@@ -0,0 +1,602 @@ +#! /usr/bin/python3 +# vim: et ts=4 sw=4 +# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +import argparse +import re +import sys +from os import environ, getcwd, makedirs, remove +from os.path import abspath, exists, isdir, join +from shutil import rmtree +from tempfile import mkdtemp + +INTERP_VERSION_RE = re.compile(r'^python(?P<version>3\.\d+)(?P<dbg>-dbg)?$') +logging.basicConfig(format='%(levelname).1s: pybuild ' + '%(module)s:%(lineno)d: %(message)s') +log = logging.getLogger('dhpython') + + +def main(cfg): + log.debug('cfg: %s', cfg) + from dhpython import build, PKG_PREFIX_MAP + from dhpython.debhelper import DebHelper, build_options + from dhpython.version import Version, build_sorted, get_requested_versions + from dhpython.interpreter import Interpreter + from dhpython.tools import execute, move_matching_files + + if cfg.list_systems: + for name, Plugin in sorted(build.plugins.items()): + print(name, '\t', Plugin.DESCRIPTION) + exit(0) + + nocheck = False + if 'DEB_BUILD_OPTIONS' in environ: + nocheck = 'nocheck' in environ['DEB_BUILD_OPTIONS'] + if not nocheck and 'DEB_BUILD_PROFILES' in environ: + nocheck = 'nocheck' in environ['DEB_BUILD_PROFILES'] + + env = environ.copy() + # set some defaults in environ to make the build reproducible + env.setdefault('LC_ALL', 'C.UTF-8') + env.setdefault('CCACHE_DIR', abspath('.pybuild/ccache')) + env.setdefault('no_proxy', 'localhost') + if 'http_proxy' not in env: + env['http_proxy'] = 'http://127.0.0.1:9/' + elif not env['http_proxy']: + del env['http_proxy'] # some tools don't like empty var. + if 'https_proxy' not in env: + env['https_proxy'] = 'https://127.0.0.1:9/' + elif not env['https_proxy']: + del env['https_proxy'] # some tools don't like empty var. + if 'DEB_PYTHON_INSTALL_LAYOUT' not in env: + env['DEB_PYTHON_INSTALL_LAYOUT'] = 'deb' + + arch_data = {} + if exists('/usr/bin/dpkg-architecture'): + res = execute('/usr/bin/dpkg-architecture') + for line in res['stdout'].splitlines(): + key, value = line.strip().split('=', 1) + arch_data[key] = value + + # Set _PYTHON_HOST_PLATFORM to ensure debugging symbols on, f.e. i386 + # emded a constant name regardless of the 32/64-bit kernel. + host_platform = '{DEB_HOST_ARCH_OS}-{DEB_HOST_ARCH}'.format(**arch_data) + # it's not called amd64 in Python + host_platform = host_platform.replace('amd64', 'x86_64') + env.setdefault('_PYTHON_HOST_PLATFORM', host_platform) + + if arch_data['DEB_BUILD_ARCH'] != arch_data['DEB_HOST_ARCH']: + # support cross compiling Python 3.X extensions, see #892931 + env.setdefault('_PYTHON_SYSCONFIGDATA_NAME', + '_sysconfigdata__' + arch_data["DEB_HOST_MULTIARCH"]) + + # Selected on command line? + selected_plugin = cfg.system + + # Selected by build_dep? + if not selected_plugin: + dh = DebHelper(build_options()) + for build_dep in dh.build_depends: + if build_dep.startswith('pybuild-plugin-'): + selected_plugin = build_dep.split('-', 2)[2] + break + + if selected_plugin: + certainty = 99 + Plugin = build.plugins.get(selected_plugin) + if not Plugin: + log.error('unrecognized build system: %s', selected_plugin) + exit(10) + plugin = Plugin(cfg) + context = {'ENV': env, 'args': {}, 'dir': cfg.dir} + plugin.detect(context) + else: + plugin, certainty, context = None, 0, None + for Plugin in build.plugins.values(): + try: + tmp_plugin = Plugin(cfg) + except Exception as err: + log.warn('cannot initialize %s plugin: %s', Plugin.NAME, + err, exc_info=cfg.verbose) + continue + tmp_context = {'ENV': env, 'args': {}, 'dir': cfg.dir} + tmp_certainty = tmp_plugin.detect(tmp_context) + log.debug('Plugin %s: certainty %i', Plugin.NAME, tmp_certainty) + if tmp_certainty and tmp_certainty > certainty: + plugin, certainty, context = tmp_plugin, tmp_certainty, tmp_context + del Plugin + if not plugin: + log.error('cannot detect build system, please use --system option' + ' or set PYBUILD_SYSTEM env. variable') + exit(11) + + if plugin.SUPPORTED_INTERPRETERS is not True: + # if versioned interpreter was requested and selected plugin lists + # versioned ones as supported: extend list of supported interpreters + # with this interpreter + tpls = {i for i in plugin.SUPPORTED_INTERPRETERS if '{version}' in i} + if tpls: + for ipreter in cfg.interpreter: + m = INTERP_VERSION_RE.match(ipreter) + if m: + ver = m.group('version') + updated = set(tpl.format(version=ver) for tpl in tpls) + updated and plugin.SUPPORTED_INTERPRETERS.update(updated) + + for interpreter in cfg.interpreter: + if plugin.SUPPORTED_INTERPRETERS is not True and interpreter not in plugin.SUPPORTED_INTERPRETERS: + log.error('interpreter %s not supported by %s', interpreter, plugin) + exit(12) + log.debug('detected build system: %s (certainty: %s%%)', plugin.NAME, certainty) + + if cfg.detect_only: + if not cfg.really_quiet: + print(plugin.NAME) + exit(0) + + versions = cfg.versions + if not versions: + if len(cfg.interpreter) == 1: + i = cfg.interpreter[0] + m = INTERP_VERSION_RE.match(i) + if m: + log.debug('defaulting to version hardcoded in interpreter name') + versions = [m.group('version')] + else: + IMAP = {v: k for k, v in PKG_PREFIX_MAP.items()} + if i in IMAP: + versions = build_sorted(get_requested_versions( + IMAP[i], available=True), impl=IMAP[i]) + if versions and '{version}' not in i: + versions = versions[-1:] # last one, the default one + if not versions: # still no luck + log.debug('defaulting to all supported Python 3.X versions') + versions = build_sorted(get_requested_versions( + 'cpython3', available=True), impl='cpython3') + versions = [Version(v) for v in versions] + + def get_option(name, interpreter=None, version=None, default=None): + if interpreter: + # try PYBUILD_NAME_python3.3-dbg (or hardcoded interpreter) + i = interpreter.format(version=version or '') + opt = "PYBUILD_{}_{}".format(name.upper(), i) + if opt in environ: + return environ[opt] + # try PYBUILD_NAME_python3-dbg (if not checked above) + if '{version}' in interpreter and version: + i = interpreter.format(version=version.major) + opt = "PYBUILD_{}_{}".format(name.upper(), i) + if opt in environ: + return environ[opt] + # try PYBUILD_NAME + opt = "PYBUILD_{}".format(name.upper()) + if opt in environ: + return environ[opt] + # try command line args + return getattr(cfg, name, default) or default + + def get_args(context, step, version, interpreter): + i = interpreter.format(version=version) + ipreter = Interpreter(i) + + home_dir = [ipreter.impl, str(version)] + if ipreter.debug: + home_dir.append('dbg') + if cfg.name: + home_dir.append(cfg.name) + if cfg.autopkgtest_only: + base_dir = environ.get('AUTOPKGTEST_TMP') + if not base_dir: + base_dir = mkdtemp(prefix='pybuild-autopkgtest-') + else: + base_dir = '.pybuild/{}' + home_dir = base_dir.format('_'.join(home_dir)) + + build_dir = get_option('build_dir', interpreter, version, + default=join(home_dir, 'build')) + + destdir = context['destdir'].format(version=version, interpreter=i) + if cfg.name: + package = ipreter.suggest_pkg_name(cfg.name) + else: + package = 'PYBUILD_NAME_not_set' + if cfg.name and destdir.rstrip('/').endswith('debian/tmp'): + destdir = "debian/{}".format(package) + destdir = abspath(destdir) + + args = dict(context['args']) + args.update({ + 'package': package, + 'interpreter': ipreter, + 'version': version, + 'args': get_option("%s_args" % step, interpreter, version, ''), + 'dir': abspath(context['dir'].format(version=version, interpreter=i)), + 'destdir': destdir, + 'build_dir': abspath(build_dir.format(version=version, interpreter=i)), + # versioned dist-packages even for Python 3.X - dh_python3 will fix it later + # (and will have a chance to compare files) + 'install_dir': get_option('install_dir', interpreter, version, + '/usr/lib/python{version}/dist-packages' + ).format(version=version, interpreter=i), + 'home_dir': abspath(home_dir)}) + if interpreter == 'pypy': + args['install_dir'] = '/usr/lib/pypy/dist-packages/' + env = dict(args.get('ENV', {})) + pp = env.get('PYTHONPATH', context['ENV'].get('PYTHONPATH')) + pp = pp.split(':') if pp else [] + if step in {'build', 'test', 'autopkgtest'}: + if step in {'test', 'autopkgtest'}: + args['test_dir'] = join(args['destdir'], args['install_dir'].lstrip('/')) + if args['test_dir'] not in pp: + pp.append(args['test_dir']) + if args['build_dir'] not in pp: + pp.append(args['build_dir']) + # cross compilation support for Python 2.x + if (version.major == 2 and + arch_data.get('DEB_BUILD_ARCH') != arch_data.get('DEB_HOST_ARCH')): + pp.insert(0, ('/usr/lib/python{0}/plat-{1[DEB_HOST_MULTIARCH]}' + ).format(version, arch_data)) + env['PYTHONPATH'] = ':'.join(pp) + # cross compilation support for Python <= 3.8 (see above) + if version.major == 3: + name = '_PYTHON_SYSCONFIGDATA_NAME' + value = env.get(name, context['ENV'].get(name, '')) + if version << '3.8' and value.startswith('_sysconfigdata_')\ + and not value.startswith('_sysconfigdata_m'): + value = env[name] = "_sysconfigdata_m%s" % value[15:] + # update default from main() for -dbg interpreter + if value and ipreter.debug and not value.startswith('_sysconfigdata_d'): + env[name] = "_sysconfigdata_d%s" % value[15:] + args['ENV'] = env + + if not exists(args['build_dir']): + makedirs(args['build_dir']) + + return args + + def is_disabled(step, interpreter, version): + i = interpreter + prefix = "{}/".format(step) + disabled = (get_option('disable', i, version) or '').split() + for item in disabled: + if item in (step, '1'): + log.debug('disabling {} step for {} {}'.format(step, i, version)) + return True + if item.startswith(prefix): + disabled.append(item[len(prefix):]) + if i in disabled or str(version) in disabled or \ + i.format(version=version) in disabled or \ + i.format(version=version.major) in disabled: + log.debug('disabling {} step for {} {}'.format(step, i, version)) + return True + return False + + def run(func, interpreter, version, context): + step = func.__func__.__name__ + args = get_args(context, step, version, interpreter) + env = dict(context['ENV']) + if 'ENV' in args: + env.update(args['ENV']) + + before_cmd = get_option('before_{}'.format(step), interpreter, version) + if before_cmd: + if cfg.quiet: + log_file = join(args['home_dir'], 'before_{}_cmd.log'.format(step)) + else: + log_file = False + command = before_cmd.format(**args) + log.info(command) + output = execute(command, context['dir'], env, log_file) + if output['returncode'] != 0: + msg = 'exit code={}: {}'.format(output['returncode'], command) + raise Exception(msg) + + fpath = join(args['home_dir'], 'testfiles_to_rm_before_install') + if step == 'install' and exists(fpath): + with open(fpath) as fp: + for line in fp: + path = line.strip('\n') + if exists(path): + if isdir(path): + rmtree(path) + else: + remove(path) + remove(fpath) + result = func(context, args) + + after_cmd = get_option('after_{}'.format(step), interpreter, version) + if after_cmd: + if cfg.quiet: + log_file = join(args['home_dir'], 'after_{}_cmd.log'.format(step)) + else: + log_file = False + command = after_cmd.format(**args) + log.info(command) + output = execute(command, context['dir'], env, log_file) + if output['returncode'] != 0: + msg = 'exit code={}: {}'.format(output['returncode'], command) + raise Exception(msg) + return result + + def move_to_ext_destdir(i, version, context): + """Move built C extensions from the general destdir to ext_destdir""" + args = get_args(context, 'install', version, interpreter) + ext_destdir = get_option('ext_destdir', i, version) + if ext_destdir: + move_matching_files(args['destdir'], ext_destdir, + get_option('ext_pattern', i, version), + get_option('ext_sub_pattern', i, version), + get_option('ext_sub_repl', i, version)) + + func = None + if cfg.clean_only: + func = plugin.clean + elif cfg.configure_only: + func = plugin.configure + elif cfg.build_only: + func = plugin.build + elif cfg.install_only: + func = plugin.install + elif cfg.test_only: + func = plugin.test + elif cfg.autopkgtest_only: + func = plugin.test + elif cfg.print_args: + func = plugin.print_args + + ### one function for each interpreter at a time mode ### + if func: + step = func.__func__.__name__ + if step == 'test' and nocheck: + exit(0) + failure = False + for i in cfg.interpreter: + ipreter = Interpreter(interpreter.format(version=versions[0])) + iversions = build_sorted(versions, impl=ipreter.impl) + if '{version}' not in i and len(versions) > 1: + log.info('limiting Python versions to %s due to missing {version}' + ' in interpreter string', str(versions[-1])) + iversions = versions[-1:] # just the default or closest to default + for version in iversions: + if is_disabled(step, i, version): + continue + c = dict(context) + c['dir'] = get_option('dir', i, version, cfg.dir) + c['destdir'] = get_option('destdir', i, version, cfg.destdir) + try: + run(func, i, version, c) + except Exception as err: + log.error('%s: plugin %s failed with: %s', + step, plugin.NAME, err, exc_info=cfg.verbose) + # try to build/test other interpreters/versions even if + # one of them fails to make build logs more verbose: + failure = True + if step not in ('build', 'test', 'autopkgtest'): + exit(13) + if step == 'install': + move_to_ext_destdir(i, version, c) + if failure: + # exit with a non-zero return code if at least one build/test failed + exit(13) + exit(0) + + ### all functions for interpreters in batches mode ### + try: + context_map = {} + for i in cfg.interpreter: + ipreter = Interpreter(interpreter.format(version=versions[0])) + iversions = build_sorted(versions, impl=ipreter.impl) + if '{version}' not in i and len(versions) > 1: + log.info('limiting Python versions to %s due to missing {version}' + ' in interpreter string', str(versions[-1])) + iversions = versions[-1:] # just the default or closest to default + for version in iversions: + key = (i, version) + if key in context_map: + c = context_map[key] + else: + c = dict(context) + c['dir'] = get_option('dir', i, version, cfg.dir) + c['destdir'] = get_option('destdir', i, version, cfg.destdir) + context_map[key] = c + + if not is_disabled('clean', i, version): + run(plugin.clean, i, version, c) + if not is_disabled('configure', i, version): + run(plugin.configure, i, version, c) + if not is_disabled('build', i, version): + run(plugin.build, i, version, c) + if not is_disabled('install', i, version): + run(plugin.install, i, version, c) + move_to_ext_destdir(i, version, c) + if not nocheck and not is_disabled('test', i, version): + run(plugin.test, i, version, c) + except Exception as err: + log.error('plugin %s failed: %s', plugin.NAME, err, + exc_info=cfg.verbose) + exit(14) + + +def parse_args(argv): + usage = '%(prog)s [ACTION] [BUILD SYSTEM ARGS] [DIRECTORIES] [OPTIONS]' + parser = argparse.ArgumentParser(usage=usage) + parser.add_argument('-v', '--verbose', action='store_true', + default=environ.get('PYBUILD_VERBOSE') == '1', + help='turn verbose mode on') + parser.add_argument('-q', '--quiet', action='store_true', + default=environ.get('PYBUILD_QUIET') == '1', + help='doesn\'t show external command\'s output') + parser.add_argument('-qq', '--really-quiet', action='store_true', + default=environ.get('PYBUILD_RQUIET') == '1', + help='be quiet') + parser.add_argument('--version', action='version', version='%(prog)s DEVELV') + + action = parser.add_argument_group('ACTION', '''The default is to build, + install and test the library using detected build system version by + version. Selecting one of following actions, will invoke given action + for all versions - one by one - which (contrary to the default action) + in some build systems can overwrite previous results.''') + action.add_argument('--detect', action='store_true', dest='detect_only', + help='return the name of detected build system') + action.add_argument('--clean', action='store_true', dest='clean_only', + help='clean files using auto-detected build system specific methods') + action.add_argument('--configure', action='store_true', dest='configure_only', + help='invoke configure step for all requested Python versions') + action.add_argument('--build', action='store_true', dest='build_only', + help='invoke build step for all requested Python versions') + action.add_argument('--install', action='store_true', dest='install_only', + help='invoke install step for all requested Python versions') + action.add_argument('--test', action='store_true', dest='test_only', + help='invoke tests for auto-detected build system') + action.add_argument('--autopkgtest', action='store_true', dest='autopkgtest_only', + help='invoke autopkgtests for auto-detected build system') + action.add_argument('--list-systems', action='store_true', + help='list available build systems and exit') + action.add_argument('--print', action='append', dest='print_args', + help="print pybuild's internal parameters") + + arguments = parser.add_argument_group('BUILD SYSTEM ARGS', ''' + Additional arguments passed to the build system. + --system=custom requires complete command.''') + arguments.add_argument('--before-clean', metavar='CMD', + help='invoked before the clean command') + arguments.add_argument('--clean-args', metavar='ARGS') + arguments.add_argument('--after-clean', metavar='CMD', + help='invoked after the clean command') + + arguments.add_argument('--before-configure', metavar='CMD', + help='invoked before the configure command') + arguments.add_argument('--configure-args', metavar='ARGS') + arguments.add_argument('--after-configure', metavar='CMD', + help='invoked after the configure command') + + arguments.add_argument('--before-build', metavar='CMD', + help='invoked before the build command') + arguments.add_argument('--build-args', metavar='ARGS') + arguments.add_argument('--after-build', metavar='CMD', + help='invoked after the build command') + + arguments.add_argument('--before-install', metavar='CMD', + help='invoked before the install command') + arguments.add_argument('--install-args', metavar='ARGS') + arguments.add_argument('--after-install', metavar='CMD', + help='invoked after the install command') + + arguments.add_argument('--before-test', metavar='CMD', + help='invoked before the test command') + arguments.add_argument('--test-args', metavar='ARGS') + arguments.add_argument('--after-test', metavar='CMD', + help='invoked after the test command') + + tests = parser.add_argument_group('TESTS', '''\ + unittest\'s discover is used by default (if available)''') + tests.add_argument('--test-nose', action='store_true', + default=environ.get('PYBUILD_TEST_NOSE') == '1', + help='use nose module in --test step') + tests.add_argument('--test-nose2', action='store_true', + default=environ.get('PYBUILD_TEST_NOSE2') == '1', + help='use nose2 module in --test step') + tests.add_argument('--test-pytest', action='store_true', + default=environ.get('PYBUILD_TEST_PYTEST') == '1', + help='use pytest module in --test step') + tests.add_argument('--test-tox', action='store_true', + default=environ.get('PYBUILD_TEST_TOX') == '1', + help='use tox in --test step') + tests.add_argument('--test-custom', action='store_true', + default=environ.get('PYBUILD_TEST_CUSTOM') == '1', + help='use custom command in --test step') + + dirs = parser.add_argument_group('DIRECTORIES') + dirs.add_argument('-d', '--dir', action='store', metavar='DIR', + default=environ.get('PYBUILD_DIR', getcwd()), + help='source files directory - base for other relative dirs [default: CWD]') + dirs.add_argument('--dest-dir', action='store', metavar='DIR', dest='destdir', + default=environ.get('DESTDIR', 'debian/tmp'), + help='destination directory [default: debian/tmp]') + dirs.add_argument('--ext-dest-dir', action='store', metavar='DIR', dest='ext_destdir', + default=environ.get('PYBUILD_EXT_DESTDIR'), + help='destination directory for .so files') + dirs.add_argument('--ext-pattern', action='store', metavar='PATTERN', + default=environ.get('PYBUILD_EXT_PATTERN', r'\.so(\.[^/]*)?$'), + help='regular expression for files that should be moved' + ' if --ext-dest-dir is set [default: .so files]') + dirs.add_argument('--ext-sub-pattern', action='store', metavar='PATTERN', + default=environ.get('PYBUILD_EXT_SUB_PATTERN'), + help='pattern to change --ext-pattern\'s filename or path') + dirs.add_argument('--ext-sub-repl', action='store', metavar='PATTERN', + default=environ.get('PYBUILD_EXT_SUB_REPL'), + help='replacement for match from --ext-sub-pattern,' + ' empty string by default') + dirs.add_argument('--install-dir', action='store', metavar='DIR', + help='installation directory [default: .../dist-packages]') + dirs.add_argument('--name', action='store', + default=environ.get('PYBUILD_NAME'), + help='use this name to guess destination directories') + + limit = parser.add_argument_group('LIMITATIONS') + limit.add_argument('-s', '--system', + default=environ.get('PYBUILD_SYSTEM'), + help='select a build system [default: auto-detection]') + limit.add_argument('-p', '--pyver', action='append', dest='versions', + help='''build for Python VERSION. + This option can be used multiple times + [default: all supported Python 3.X versions]''') + limit.add_argument('-i', '--interpreter', action='append', + help='change interpreter [default: python{version}]') + limit.add_argument('--disable', metavar='ITEMS', + help='disable action, interpreter or version') + + args = parser.parse_args() + if not args.interpreter: + args.interpreter = environ.get('PYBUILD_INTERPRETERS', 'python{version}').split() + if not args.versions: + args.versions = environ.get('PYBUILD_VERSIONS', '').split() + else: + # add support for -p `pyversions -rv` + versions = [] + for version in args.versions: + versions.extend(version.split()) + args.versions = versions + + if args.test_nose or args.test_nose2 or args.test_pytest or args.test_tox\ + or args.test_custom or args.system == 'custom': + args.custom_tests = True + else: + args.custom_tests = False + + return args + + +if __name__ == '__main__': + cfg = parse_args(sys.argv) + if cfg.really_quiet: + cfg.quiet = True + log.setLevel(logging.CRITICAL) + elif cfg.verbose: + log.setLevel(logging.DEBUG) + else: + log.setLevel(logging.INFO) + log.debug('version: DEVELV') + log.debug(sys.argv) + main(cfg) + # let dh/cdbs clean the .pybuild dir + # rmtree(join(cfg.dir, '.pybuild')) diff --git a/pybuild-autopkgtest b/pybuild-autopkgtest new file mode 100755 index 0000000..5b3af8d --- /dev/null +++ b/pybuild-autopkgtest @@ -0,0 +1,68 @@ +#! /usr/bin/env perl +# vim: et ts=4 sw=4 +# Copyright © 2021 Antonio Terceiro <terceiro@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +use strict; +use warnings; +use File::Basename; +use File::Temp qw( tempdir ); +use Debian::Debhelper::Buildsystem::pybuild; +use Debian::Debhelper::Dh_Lib qw(doit); + +sub main { + my $tmpdir = tempdir( CLEANUP => 1); + my $run = "${tmpdir}/run"; + open(RUN, ">", $run) or die($!); + print(RUN "#!/usr/bin/make -f\n"); + print(RUN "include debian/rules\n"); + print(RUN "pybuild-autopkgtest:\n"); + printf(RUN "\tpybuild-autopkgtest\n"); + close(RUN); + chmod(0755, $run); + $ENV{PYBUILD_AUTOPKGTEST} = "1"; + if (system("grep -q ^before-pybuild-autopkgtest: debian/rules") == 0) { + doit($run, "before-pybuild-autopkgtest"); + } + doit($run, "pybuild-autopkgtest"); + if (system("grep -q ^after-pybuild-autopkgtest: debian/rules") == 0) { + doit($run, "after-pybuild-autopkgtest"); + } +} + +sub child { + # The child inherits the environment defined in debian/rules + my $dh = Debian::Debhelper::Buildsystem::pybuild->new(); + foreach my $command ($dh->pybuild_commands('autopkgtest')) { + doit(@$command); + } +} + +if (scalar(@ARGV) > 0) { + my $prog = basename($0); + print STDERR "usage: ${prog}\n"; + exit(1); +} + +if (defined $ENV{PYBUILD_AUTOPKGTEST}) { + child; +} else { + main; +} diff --git a/pybuild-autopkgtest.rst b/pybuild-autopkgtest.rst new file mode 100644 index 0000000..76da53d --- /dev/null +++ b/pybuild-autopkgtest.rst @@ -0,0 +1,109 @@ +===================== + pybuild-autopkgtest +===================== + +---------------------------------------------------------------------------------------------------- +invokes the test suite against requested Python versions and installed packages +---------------------------------------------------------------------------------------------------- + +:Manual section: 1 +:Author: Antonio Terceiro, 2021 + +SYNOPSIS +======== + pybuild-autopkgtest + +OPTIONS +======= + +`pybuild-autopkgtest` takes no options or arguments. All configuration is done +via the same mechanisms you use to control `pybuild` itself, namely by having +build dependencies on the right packages, or by setting `PYBUILD_*` environment +variables in `debian/rules`. + +DESCRIPTION +=========== + +`pybuild-autopkgtest` is an autopkgtest test runner that reuses all the +`pybuild` infrastructure to run tests against installed packages, as expected +by autopkgtest. To enable `pybuild-autopkgtest` for your package, you need to +add **Testsuite: autopkgtest-pkg-pybuild** to the source stanza in +`debian/control`. This will cause autodep8(1) to produce the correct contents +for `debian/tests/control`. + +`pybuild-autopkgtest` will run the tests in exactly the same way as `pybuild` +will during the build, with the exception that the tests are not run in the +build directory. The test files are copied to a temporary directory, so that +the tests will run against the installed Python modules, and not against +anything in the source tree. + +All the pybuild infrastructure is used, so for most packages you don't need to +add any extra code configure for `pybuild-autopkgtest`. For example, just +having a `python3-pytest` as a build dependency is enough to make the test +runner use `pytest` to run the tests. + +The tests are executed via a temporary makefile that includes `debian/rules` +from the package, so that any environment variables defined there will also be +available during autopkgtest, including but not limited to `PYBUILD_*` +variables for configuring the behavior of `pybuild` itself. + +ADOPTING PYBUILD-AUTOPKGTEST +============================ + +Since `pybuild-autopkgtest` reuses environment variables set in `debian/rules`, +migrating packages to use `pybuild-autopkgtest` should not require much effort. + +You might have a `debian/tests/control` that duplicates what +`pybuild-autopkgtest` already does, e.g. copying the test files to a temporary +directory, changing to it, and running the tests from there. Such test entries +can usually be removed in favor of adding **Testsuite: +autopkgtest-pkg-pybuild** to `debian/control`. + +In general, you want to move any test-related command line arguments to pybuild +into environment variables in `debian/rules`. + +You can also have specialized, manually-written test cases, alongside the ones +autogenerated by `autodep8`. For this, both set **Testsuite: +autopkgtest-pkg-pybuild** in `debian/control` and keep your custom tests in +`debian/tests/control`. + +VARYING BEHAVIOR UNDER AUTOPKGTEST +================================== + +Ideally, the behavior of the tests should be the same during the build and +under autopkgtest, except for the fact that during autopkgtest the tests should +load the code from the installed package. `pybuild-autopkgtest` should do this +correctly most of the time. + +There are situations, however, in which you need a slightly different behavior +during the autopkgtest run. There are a few mechanisms to support that: + +- `pybuild-autopkgtest` sets the `PYBUILD_AUTOPKGTEST` environment variable to + `1` during the test run. This way, you can add conditional behavior in + `debian/rules`. +- Before and after running the tests, `pybuild-autopkgtest` will call the + `debian/rules` targets `before-pybuild-autopkgtest` and + `after-pybuild-autopkgtest`, respectively, if they exist. + +SAMPLE TEST CONTROL FILE +======================== + +The control file produced by autodep8(1) looks like this:: + + Test-Command: pybuild-autopkgtest + Depends: @, @builddeps@, + Restrictions: allow-stderr, skippable, + Features: test-name=pybuild-autopkgtest + +You should never need to hardcode this in `debian/tests/control`. You can add +extra items to `Restrictions` and `Depends` by providing a configuration file +for `autodep8` (`debian/tests/autopkgtest-pkg-pybuild.conf`) like this:: + + extra_depends=foo, bar + extra_restrictions=isolation-container, breaks-testbed + +SEE ALSO +======== +* pybuild(1) +* autopkgtest(1) +* autodep8(1) diff --git a/pybuild.rst b/pybuild.rst new file mode 100644 index 0000000..f60db1f --- /dev/null +++ b/pybuild.rst @@ -0,0 +1,306 @@ +========= + pybuild +========= + +---------------------------------------------------------------------------------------------------- +invokes various build systems for requested Python versions in order to build modules and extensions +---------------------------------------------------------------------------------------------------- + +:Manual section: 1 +:Author: Piotr Ożarowski, 2012-2019 + +SYNOPSIS +======== + pybuild [ACTION] [BUILD SYSTEM ARGUMENTS] [DIRECTORIES] [OPTIONS] + +DEBHELPER COMMAND SEQUENCER INTEGRATION +======================================= +* build depend on `dh-python`, +* build depend on all supported Python interpreters, pybuild will use it to create + a list of interpreters to build for. + Recognized dependencies: + + - `python3-all-dev` - for Python extensions that work with Python 3.X interpreters, + - `python3-all-dbg` - as above, add this one if you're building -dbg packages, + - `python3-all` - for Python modules that work with Python 3.X interpreters, + - `python3-dev` - builds an extension for default Python 3.X interpreter + (useful for private extensions, use python3-all-dev for public ones), + - `python3` - as above, used if headers files are not needed to build private module, + +* add `--buildsystem=pybuild` to dh's arguments in debian/rules, +* if more than one binary package is build: + add debian/python3-foo.install files, or + `export PYBUILD_NAME=modulename` (modulename will be used to guess binary + package prefixes), or + `export PYBUILD_DESTDIR` env. variables in debian/rules +* add `--with=python3` to dh's arguments in debian/rules + (see proper helper's manpage for more details) or add `dh-sequence-python3` + to Build-Depends + +debian/rules file example:: + + #! /usr/bin/make -f + export PYBUILD_NAME=foo + %: + dh $@ --with python3 --buildsystem=pybuild + +OPTIONS +======= + Most options can be set (in addition to command line) via environment + variables. PyBuild will check: + + * PYBUILD_OPTION_VERSIONED_INTERPRETER (f.e. PYBUILD_CLEAN_ARGS_python3.2) + * PYBUILD_OPTION_INTERPRETER (f.e. PYBUILD_CONFIGURE_ARGS_python3-dbg) + * PYBUILD_OPTION (f.e. PYBUILD_INSTALL_ARGS) + +optional arguments +------------------ + -h, --help show this help message and exit + -v, --verbose turn verbose mode on + -q, --quiet doesn't show external command's output + -qq, --really-quiet be quiet + --version show program's version number and exit + +ACTION +------ + The default is to build, install and test the library using detected build + system version by version. Selecting one of following actions, will invoke + given action for all versions - one by one - which (contrary to the default + action) in some build systems can overwrite previous results. + + --detect + return the name of detected build system + --clean + clean files using auto-detected build system specific methods + --configure + invoke configure step for all requested Python versions + --build + invoke build step for all requested Python versions + --install + invoke install step for all requested Python versions + --test + invoke tests for auto-detected build system + --list-systems + list available build systems and exit + --print + print pybuild's internal parameters + +TESTS +----- + unittest's discover from standard library is used in test step by default. + + --test-nose + use nose module in test step, remember to add python-nose and/or + python3-nose to Build-Depends + --test-nose2 + use nose2 module in test step, remember to add python-nose2 and/or + python3-nose2 to Build-Depends + --test-pytest + use pytest module in test step, remember to add python-pytest and/or + python3-pytest to Build-Depends + --test-tox + use tox command in test step, remember to add tox + to Build-Depends. Requires tox.ini file + --test-custom + use a custom command in the test step. The full test command is then + specified with `--test-args` or by setting the `PYBUILD_TEST_ARGS` + environment variable. Remember to add any needed packages to run the + tests to Build-Depends. + + +testfiles +~~~~~~~~~ + Tests are invoked from within build directory to make sure newly built + files are tested instead of source files. If test suite requires other files + in this directory, you can list them in `debian/pybuild.testfiles` file + (you can also use `debian/pybuild_pythonX.testfiles` or + `debian/pybuild_pythonX.Y.testfiles`) and files listed there will be copied + before test step and removed before install step. + By default only `test` and `tests` directories are copied to build directory. + +BUILD SYSTEM ARGUMENTS +---------------------- + Additional arguments passed to the build system. + --system=custom requires complete command in --foo-args parameters. + + --before-clean COMMAND + invoked before the clean command + --clean-args ARGUMENTS + arguments added to clean command generated by build system plugin + --after-clean COMMAND + invoked after the clean command + --before-configure COMMAND + invoked before the configure command + --configure-args ARGUMENTS + arguments added to configure command generated by build system plugin + --after-configure COMMAND + invoked after the configure command + --before-build COMMAND + invoked before the build command + --build-args ARGUMENTS + arguments added to build command generated by build system plugin + --after-build COMMAND + invoked after the build command + --before-install COMMAND + invoked before the install command + --install-args ARGUMENTS + arguments added to install command generated by build system plugin + --after-install COMMAND + invoked after the install command + --before-test COMMAND + invoked before the test command + --test-args ARGUMENTS + arguments added to test command generated by build system plugin + --after-test COMMAND + invoked after the test command + +variables that can be used in `ARGUMENTS` and `COMMAND` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* `{version}` will be replaced with current Python version, + you can also use `{version.major}`, `{version.minor}`, etc. +* `{interpreter}` will be replaced with current interpreter, + you can also use `{interpreter.include_dir}` +* `{dir}` will be replaced with sources directory, +* `{destdir}` will be replaced with destination directory, +* `{home_dir}` will be replaced with temporary HOME directory, + where plugins can keep their data + (.pybuild/interpreter_version/ by default), +* `{build_dir}` will be replaced with build directory +* `{install_dir}` will be replaced with install directory. +* `{package}` will be replaced with suggested package name, + if --name (or PYBUILD_NAME) is set to `foo`, this variable + will be replaced with `python3-foo`. + +DIRECTORIES +----------- + -d DIR, --dir DIR + set source files directory - base for other relative dirs + [by default: current working directory] + --dest-dir DIR + set destination directory [default: debian/tmp] + --ext-dest-dir DIR + set destination directory for .so files + --ext-pattern PATTERN + regular expression for files that should be moved if --ext-dest-dir is set + [default: `\.so(\.[^/]*)?$`] + --ext-sub-pattern PATTERN + regular expression for part of path/filename matched in --ext-pattern + that should be removed or replaced with --ext-sub-repl + --ext-sub-repl PATTERN + replacement for matches in --ext-sub-pattern + --install-dir DIR + set installation directory [default: .../dist-packages] + --name NAME + use this name to guess destination directories + ("foo" sets debian/python3-foo) + This overrides --dest-dir. + +variables that can be used in `DIR` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* `{version}` will be replaced with current Python version, +* `{interpreter}` will be replaced with selected interpreter. + +LIMITATIONS +----------- + -s SYSTEM, --system SYSTEM + select a build system [default: auto-detection] + -p VERSIONS, --pyver VERSIONS + build for Python VERSIONS. This option can be used multiple times. + Versions can be separated by space character. + The default is all Python 3.X supported versions. + -i INTERPRETER, --interpreter INTERPRETER + change interpreter [default: python{version}] + --disable ITEMS + disable action, interpreter, version or any mix of them. + Note that f.e. python3 and python3-dbg are two different interpreters, + --disable test/python3 doesn't disable python3-dbg's tests. + +disable examples +~~~~~~~~~~~~~~~~ +* `--disable test/python3.9-dbg` - disables tests for python3.9-dbg +* `--disable '3.8 3.9'` - disables all actions for version 3.8 and 3.9 +* `PYBUILD_DISABLE=python3.9` - disables all actions for Python 3.9 +* `PYBUILD_DISABLE_python3.3=test` - disables tests for Python 3.3 +* `PYBUILD_DISABLE=test/python3.3` - same as above +* `PYBUILD_DISABLE=configure/python3 3.2` - disables configure + action for all python3 interpreters, and all actions for version 3.2 + + +PLUGINS +------- +pybuild supports multiple build system plugins. By default it is +automatically selected. These systems are currently supported:: + +* distutils (most commonly used) +* cmake +* flit (deprecated) +* pyproject +* custom + +flit plugin +~~~~~~~~~~~ +The flit plugin is deprecated, please use the pyproject plugin instead. + +The flit plugin can be used to build Debian packages based on PEP 517 +metadata in `pyproject.toml` when flit is the upstream build system. These +can be identified by the presence of a `build-backend = "flit_core.buildapi"` +element in `pyproject.toml`. The flit plugin only supports python3. To use +this plugin:: + +* build depend on `flit` and either +* build depend on `python3-tomli` so flit can be automatically selected or +* add `export PYBUILD_SYSTEM=flit` to debian/rules to manually select + +debian/rules file example:: + + #! /usr/bin/make -f + export PYBUILD_NAME=foo + export PYBUILD_SYSTEM=flit (needed if python3-tomli is not installed) + %: + dh $@ --with python3 --buildsystem=pybuild + +pyproject +~~~~~~~~~ +The pyproject plugin drives the new PEP-517 standard interface for +building Python packages, upstream. This is configured via +`pyproject.toml`. +This plugin is expected to replace the distutils and flit plugins in the +future. +The entry points generated by the package are created during the build step +(other plugins make the entry points during the install step); the entry +points are available in PATH during the test step, permitting them to be +called from tests. + +To use this plugin: + +* build depend on `pybuild-plugin-pyproject` as well as any build tools + specified by upstream in `pyproject.toml`. + +ENVIRONMENT +=========== + +As described above in OPTIONS, pybuild can be configured by `PYBUILD_` +prefixed environment variables. + +Tests are skipped if `nocheck` is in the `DEB_BUILD_OPTIONS` or +`DEB_BUILD_PROFILES` environment variables. + +`DESTDIR` provides a default a default value to the `--dest-dir` option. + +Pybuild will export `http_proxy=http://127.0.0.1:9/`, +`https_proxy=https://127.0.0.1:9/`, and `no_proxy=localhost` to +hopefully block attempts by the package's build-system to access the +Internet. +If network access to a loopback interface is needed and blocked by this, +export empty `http_proxy` and `https_proxy` variables before calling +pybuild. + +If not set, `LC_ALL`, `CCACHE_DIR`, `DEB_PYTHON_INSTALL_LAYOUT`, +`_PYTHON_HOST_PLATFORM`, `_PYTHON_SYSCONFIGDATA_NAME`, will all be set +to appropriate values, before calling the package's build script. + +SEE ALSO +======== +* dh_python3(1) +* https://wiki.debian.org/Python/Pybuild +* http://deb.li/pybuild - most recent version of this document |