diff options
Diffstat (limited to 'pybuild')
| -rwxr-xr-x | pybuild | 602 |
1 files changed, 602 insertions, 0 deletions
@@ -0,0 +1,602 @@ +#! /usr/bin/python3 +# vim: et ts=4 sw=4 +# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +import argparse +import re +import sys +from os import environ, getcwd, makedirs, remove +from os.path import abspath, exists, isdir, join +from shutil import rmtree +from tempfile import mkdtemp + +INTERP_VERSION_RE = re.compile(r'^python(?P<version>3\.\d+)(?P<dbg>-dbg)?$') +logging.basicConfig(format='%(levelname).1s: pybuild ' + '%(module)s:%(lineno)d: %(message)s') +log = logging.getLogger('dhpython') + + +def main(cfg): + log.debug('cfg: %s', cfg) + from dhpython import build, PKG_PREFIX_MAP + from dhpython.debhelper import DebHelper, build_options + from dhpython.version import Version, build_sorted, get_requested_versions + from dhpython.interpreter import Interpreter + from dhpython.tools import execute, move_matching_files + + if cfg.list_systems: + for name, Plugin in sorted(build.plugins.items()): + print(name, '\t', Plugin.DESCRIPTION) + exit(0) + + nocheck = False + if 'DEB_BUILD_OPTIONS' in environ: + nocheck = 'nocheck' in environ['DEB_BUILD_OPTIONS'] + if not nocheck and 'DEB_BUILD_PROFILES' in environ: + nocheck = 'nocheck' in environ['DEB_BUILD_PROFILES'] + + env = environ.copy() + # set some defaults in environ to make the build reproducible + env.setdefault('LC_ALL', 'C.UTF-8') + env.setdefault('CCACHE_DIR', abspath('.pybuild/ccache')) + env.setdefault('no_proxy', 'localhost') + if 'http_proxy' not in env: + env['http_proxy'] = 'http://127.0.0.1:9/' + elif not env['http_proxy']: + del env['http_proxy'] # some tools don't like empty var. + if 'https_proxy' not in env: + env['https_proxy'] = 'https://127.0.0.1:9/' + elif not env['https_proxy']: + del env['https_proxy'] # some tools don't like empty var. + if 'DEB_PYTHON_INSTALL_LAYOUT' not in env: + env['DEB_PYTHON_INSTALL_LAYOUT'] = 'deb' + + arch_data = {} + if exists('/usr/bin/dpkg-architecture'): + res = execute('/usr/bin/dpkg-architecture') + for line in res['stdout'].splitlines(): + key, value = line.strip().split('=', 1) + arch_data[key] = value + + # Set _PYTHON_HOST_PLATFORM to ensure debugging symbols on, f.e. i386 + # emded a constant name regardless of the 32/64-bit kernel. + host_platform = '{DEB_HOST_ARCH_OS}-{DEB_HOST_ARCH}'.format(**arch_data) + # it's not called amd64 in Python + host_platform = host_platform.replace('amd64', 'x86_64') + env.setdefault('_PYTHON_HOST_PLATFORM', host_platform) + + if arch_data['DEB_BUILD_ARCH'] != arch_data['DEB_HOST_ARCH']: + # support cross compiling Python 3.X extensions, see #892931 + env.setdefault('_PYTHON_SYSCONFIGDATA_NAME', + '_sysconfigdata__' + arch_data["DEB_HOST_MULTIARCH"]) + + # Selected on command line? + selected_plugin = cfg.system + + # Selected by build_dep? + if not selected_plugin: + dh = DebHelper(build_options()) + for build_dep in dh.build_depends: + if build_dep.startswith('pybuild-plugin-'): + selected_plugin = build_dep.split('-', 2)[2] + break + + if selected_plugin: + certainty = 99 + Plugin = build.plugins.get(selected_plugin) + if not Plugin: + log.error('unrecognized build system: %s', selected_plugin) + exit(10) + plugin = Plugin(cfg) + context = {'ENV': env, 'args': {}, 'dir': cfg.dir} + plugin.detect(context) + else: + plugin, certainty, context = None, 0, None + for Plugin in build.plugins.values(): + try: + tmp_plugin = Plugin(cfg) + except Exception as err: + log.warn('cannot initialize %s plugin: %s', Plugin.NAME, + err, exc_info=cfg.verbose) + continue + tmp_context = {'ENV': env, 'args': {}, 'dir': cfg.dir} + tmp_certainty = tmp_plugin.detect(tmp_context) + log.debug('Plugin %s: certainty %i', Plugin.NAME, tmp_certainty) + if tmp_certainty and tmp_certainty > certainty: + plugin, certainty, context = tmp_plugin, tmp_certainty, tmp_context + del Plugin + if not plugin: + log.error('cannot detect build system, please use --system option' + ' or set PYBUILD_SYSTEM env. variable') + exit(11) + + if plugin.SUPPORTED_INTERPRETERS is not True: + # if versioned interpreter was requested and selected plugin lists + # versioned ones as supported: extend list of supported interpreters + # with this interpreter + tpls = {i for i in plugin.SUPPORTED_INTERPRETERS if '{version}' in i} + if tpls: + for ipreter in cfg.interpreter: + m = INTERP_VERSION_RE.match(ipreter) + if m: + ver = m.group('version') + updated = set(tpl.format(version=ver) for tpl in tpls) + updated and plugin.SUPPORTED_INTERPRETERS.update(updated) + + for interpreter in cfg.interpreter: + if plugin.SUPPORTED_INTERPRETERS is not True and interpreter not in plugin.SUPPORTED_INTERPRETERS: + log.error('interpreter %s not supported by %s', interpreter, plugin) + exit(12) + log.debug('detected build system: %s (certainty: %s%%)', plugin.NAME, certainty) + + if cfg.detect_only: + if not cfg.really_quiet: + print(plugin.NAME) + exit(0) + + versions = cfg.versions + if not versions: + if len(cfg.interpreter) == 1: + i = cfg.interpreter[0] + m = INTERP_VERSION_RE.match(i) + if m: + log.debug('defaulting to version hardcoded in interpreter name') + versions = [m.group('version')] + else: + IMAP = {v: k for k, v in PKG_PREFIX_MAP.items()} + if i in IMAP: + versions = build_sorted(get_requested_versions( + IMAP[i], available=True), impl=IMAP[i]) + if versions and '{version}' not in i: + versions = versions[-1:] # last one, the default one + if not versions: # still no luck + log.debug('defaulting to all supported Python 3.X versions') + versions = build_sorted(get_requested_versions( + 'cpython3', available=True), impl='cpython3') + versions = [Version(v) for v in versions] + + def get_option(name, interpreter=None, version=None, default=None): + if interpreter: + # try PYBUILD_NAME_python3.3-dbg (or hardcoded interpreter) + i = interpreter.format(version=version or '') + opt = "PYBUILD_{}_{}".format(name.upper(), i) + if opt in environ: + return environ[opt] + # try PYBUILD_NAME_python3-dbg (if not checked above) + if '{version}' in interpreter and version: + i = interpreter.format(version=version.major) + opt = "PYBUILD_{}_{}".format(name.upper(), i) + if opt in environ: + return environ[opt] + # try PYBUILD_NAME + opt = "PYBUILD_{}".format(name.upper()) + if opt in environ: + return environ[opt] + # try command line args + return getattr(cfg, name, default) or default + + def get_args(context, step, version, interpreter): + i = interpreter.format(version=version) + ipreter = Interpreter(i) + + home_dir = [ipreter.impl, str(version)] + if ipreter.debug: + home_dir.append('dbg') + if cfg.name: + home_dir.append(cfg.name) + if cfg.autopkgtest_only: + base_dir = environ.get('AUTOPKGTEST_TMP') + if not base_dir: + base_dir = mkdtemp(prefix='pybuild-autopkgtest-') + else: + base_dir = '.pybuild/{}' + home_dir = base_dir.format('_'.join(home_dir)) + + build_dir = get_option('build_dir', interpreter, version, + default=join(home_dir, 'build')) + + destdir = context['destdir'].format(version=version, interpreter=i) + if cfg.name: + package = ipreter.suggest_pkg_name(cfg.name) + else: + package = 'PYBUILD_NAME_not_set' + if cfg.name and destdir.rstrip('/').endswith('debian/tmp'): + destdir = "debian/{}".format(package) + destdir = abspath(destdir) + + args = dict(context['args']) + args.update({ + 'package': package, + 'interpreter': ipreter, + 'version': version, + 'args': get_option("%s_args" % step, interpreter, version, ''), + 'dir': abspath(context['dir'].format(version=version, interpreter=i)), + 'destdir': destdir, + 'build_dir': abspath(build_dir.format(version=version, interpreter=i)), + # versioned dist-packages even for Python 3.X - dh_python3 will fix it later + # (and will have a chance to compare files) + 'install_dir': get_option('install_dir', interpreter, version, + '/usr/lib/python{version}/dist-packages' + ).format(version=version, interpreter=i), + 'home_dir': abspath(home_dir)}) + if interpreter == 'pypy': + args['install_dir'] = '/usr/lib/pypy/dist-packages/' + env = dict(args.get('ENV', {})) + pp = env.get('PYTHONPATH', context['ENV'].get('PYTHONPATH')) + pp = pp.split(':') if pp else [] + if step in {'build', 'test', 'autopkgtest'}: + if step in {'test', 'autopkgtest'}: + args['test_dir'] = join(args['destdir'], args['install_dir'].lstrip('/')) + if args['test_dir'] not in pp: + pp.append(args['test_dir']) + if args['build_dir'] not in pp: + pp.append(args['build_dir']) + # cross compilation support for Python 2.x + if (version.major == 2 and + arch_data.get('DEB_BUILD_ARCH') != arch_data.get('DEB_HOST_ARCH')): + pp.insert(0, ('/usr/lib/python{0}/plat-{1[DEB_HOST_MULTIARCH]}' + ).format(version, arch_data)) + env['PYTHONPATH'] = ':'.join(pp) + # cross compilation support for Python <= 3.8 (see above) + if version.major == 3: + name = '_PYTHON_SYSCONFIGDATA_NAME' + value = env.get(name, context['ENV'].get(name, '')) + if version << '3.8' and value.startswith('_sysconfigdata_')\ + and not value.startswith('_sysconfigdata_m'): + value = env[name] = "_sysconfigdata_m%s" % value[15:] + # update default from main() for -dbg interpreter + if value and ipreter.debug and not value.startswith('_sysconfigdata_d'): + env[name] = "_sysconfigdata_d%s" % value[15:] + args['ENV'] = env + + if not exists(args['build_dir']): + makedirs(args['build_dir']) + + return args + + def is_disabled(step, interpreter, version): + i = interpreter + prefix = "{}/".format(step) + disabled = (get_option('disable', i, version) or '').split() + for item in disabled: + if item in (step, '1'): + log.debug('disabling {} step for {} {}'.format(step, i, version)) + return True + if item.startswith(prefix): + disabled.append(item[len(prefix):]) + if i in disabled or str(version) in disabled or \ + i.format(version=version) in disabled or \ + i.format(version=version.major) in disabled: + log.debug('disabling {} step for {} {}'.format(step, i, version)) + return True + return False + + def run(func, interpreter, version, context): + step = func.__func__.__name__ + args = get_args(context, step, version, interpreter) + env = dict(context['ENV']) + if 'ENV' in args: + env.update(args['ENV']) + + before_cmd = get_option('before_{}'.format(step), interpreter, version) + if before_cmd: + if cfg.quiet: + log_file = join(args['home_dir'], 'before_{}_cmd.log'.format(step)) + else: + log_file = False + command = before_cmd.format(**args) + log.info(command) + output = execute(command, context['dir'], env, log_file) + if output['returncode'] != 0: + msg = 'exit code={}: {}'.format(output['returncode'], command) + raise Exception(msg) + + fpath = join(args['home_dir'], 'testfiles_to_rm_before_install') + if step == 'install' and exists(fpath): + with open(fpath) as fp: + for line in fp: + path = line.strip('\n') + if exists(path): + if isdir(path): + rmtree(path) + else: + remove(path) + remove(fpath) + result = func(context, args) + + after_cmd = get_option('after_{}'.format(step), interpreter, version) + if after_cmd: + if cfg.quiet: + log_file = join(args['home_dir'], 'after_{}_cmd.log'.format(step)) + else: + log_file = False + command = after_cmd.format(**args) + log.info(command) + output = execute(command, context['dir'], env, log_file) + if output['returncode'] != 0: + msg = 'exit code={}: {}'.format(output['returncode'], command) + raise Exception(msg) + return result + + def move_to_ext_destdir(i, version, context): + """Move built C extensions from the general destdir to ext_destdir""" + args = get_args(context, 'install', version, interpreter) + ext_destdir = get_option('ext_destdir', i, version) + if ext_destdir: + move_matching_files(args['destdir'], ext_destdir, + get_option('ext_pattern', i, version), + get_option('ext_sub_pattern', i, version), + get_option('ext_sub_repl', i, version)) + + func = None + if cfg.clean_only: + func = plugin.clean + elif cfg.configure_only: + func = plugin.configure + elif cfg.build_only: + func = plugin.build + elif cfg.install_only: + func = plugin.install + elif cfg.test_only: + func = plugin.test + elif cfg.autopkgtest_only: + func = plugin.test + elif cfg.print_args: + func = plugin.print_args + + ### one function for each interpreter at a time mode ### + if func: + step = func.__func__.__name__ + if step == 'test' and nocheck: + exit(0) + failure = False + for i in cfg.interpreter: + ipreter = Interpreter(interpreter.format(version=versions[0])) + iversions = build_sorted(versions, impl=ipreter.impl) + if '{version}' not in i and len(versions) > 1: + log.info('limiting Python versions to %s due to missing {version}' + ' in interpreter string', str(versions[-1])) + iversions = versions[-1:] # just the default or closest to default + for version in iversions: + if is_disabled(step, i, version): + continue + c = dict(context) + c['dir'] = get_option('dir', i, version, cfg.dir) + c['destdir'] = get_option('destdir', i, version, cfg.destdir) + try: + run(func, i, version, c) + except Exception as err: + log.error('%s: plugin %s failed with: %s', + step, plugin.NAME, err, exc_info=cfg.verbose) + # try to build/test other interpreters/versions even if + # one of them fails to make build logs more verbose: + failure = True + if step not in ('build', 'test', 'autopkgtest'): + exit(13) + if step == 'install': + move_to_ext_destdir(i, version, c) + if failure: + # exit with a non-zero return code if at least one build/test failed + exit(13) + exit(0) + + ### all functions for interpreters in batches mode ### + try: + context_map = {} + for i in cfg.interpreter: + ipreter = Interpreter(interpreter.format(version=versions[0])) + iversions = build_sorted(versions, impl=ipreter.impl) + if '{version}' not in i and len(versions) > 1: + log.info('limiting Python versions to %s due to missing {version}' + ' in interpreter string', str(versions[-1])) + iversions = versions[-1:] # just the default or closest to default + for version in iversions: + key = (i, version) + if key in context_map: + c = context_map[key] + else: + c = dict(context) + c['dir'] = get_option('dir', i, version, cfg.dir) + c['destdir'] = get_option('destdir', i, version, cfg.destdir) + context_map[key] = c + + if not is_disabled('clean', i, version): + run(plugin.clean, i, version, c) + if not is_disabled('configure', i, version): + run(plugin.configure, i, version, c) + if not is_disabled('build', i, version): + run(plugin.build, i, version, c) + if not is_disabled('install', i, version): + run(plugin.install, i, version, c) + move_to_ext_destdir(i, version, c) + if not nocheck and not is_disabled('test', i, version): + run(plugin.test, i, version, c) + except Exception as err: + log.error('plugin %s failed: %s', plugin.NAME, err, + exc_info=cfg.verbose) + exit(14) + + +def parse_args(argv): + usage = '%(prog)s [ACTION] [BUILD SYSTEM ARGS] [DIRECTORIES] [OPTIONS]' + parser = argparse.ArgumentParser(usage=usage) + parser.add_argument('-v', '--verbose', action='store_true', + default=environ.get('PYBUILD_VERBOSE') == '1', + help='turn verbose mode on') + parser.add_argument('-q', '--quiet', action='store_true', + default=environ.get('PYBUILD_QUIET') == '1', + help='doesn\'t show external command\'s output') + parser.add_argument('-qq', '--really-quiet', action='store_true', + default=environ.get('PYBUILD_RQUIET') == '1', + help='be quiet') + parser.add_argument('--version', action='version', version='%(prog)s DEVELV') + + action = parser.add_argument_group('ACTION', '''The default is to build, + install and test the library using detected build system version by + version. Selecting one of following actions, will invoke given action + for all versions - one by one - which (contrary to the default action) + in some build systems can overwrite previous results.''') + action.add_argument('--detect', action='store_true', dest='detect_only', + help='return the name of detected build system') + action.add_argument('--clean', action='store_true', dest='clean_only', + help='clean files using auto-detected build system specific methods') + action.add_argument('--configure', action='store_true', dest='configure_only', + help='invoke configure step for all requested Python versions') + action.add_argument('--build', action='store_true', dest='build_only', + help='invoke build step for all requested Python versions') + action.add_argument('--install', action='store_true', dest='install_only', + help='invoke install step for all requested Python versions') + action.add_argument('--test', action='store_true', dest='test_only', + help='invoke tests for auto-detected build system') + action.add_argument('--autopkgtest', action='store_true', dest='autopkgtest_only', + help='invoke autopkgtests for auto-detected build system') + action.add_argument('--list-systems', action='store_true', + help='list available build systems and exit') + action.add_argument('--print', action='append', dest='print_args', + help="print pybuild's internal parameters") + + arguments = parser.add_argument_group('BUILD SYSTEM ARGS', ''' + Additional arguments passed to the build system. + --system=custom requires complete command.''') + arguments.add_argument('--before-clean', metavar='CMD', + help='invoked before the clean command') + arguments.add_argument('--clean-args', metavar='ARGS') + arguments.add_argument('--after-clean', metavar='CMD', + help='invoked after the clean command') + + arguments.add_argument('--before-configure', metavar='CMD', + help='invoked before the configure command') + arguments.add_argument('--configure-args', metavar='ARGS') + arguments.add_argument('--after-configure', metavar='CMD', + help='invoked after the configure command') + + arguments.add_argument('--before-build', metavar='CMD', + help='invoked before the build command') + arguments.add_argument('--build-args', metavar='ARGS') + arguments.add_argument('--after-build', metavar='CMD', + help='invoked after the build command') + + arguments.add_argument('--before-install', metavar='CMD', + help='invoked before the install command') + arguments.add_argument('--install-args', metavar='ARGS') + arguments.add_argument('--after-install', metavar='CMD', + help='invoked after the install command') + + arguments.add_argument('--before-test', metavar='CMD', + help='invoked before the test command') + arguments.add_argument('--test-args', metavar='ARGS') + arguments.add_argument('--after-test', metavar='CMD', + help='invoked after the test command') + + tests = parser.add_argument_group('TESTS', '''\ + unittest\'s discover is used by default (if available)''') + tests.add_argument('--test-nose', action='store_true', + default=environ.get('PYBUILD_TEST_NOSE') == '1', + help='use nose module in --test step') + tests.add_argument('--test-nose2', action='store_true', + default=environ.get('PYBUILD_TEST_NOSE2') == '1', + help='use nose2 module in --test step') + tests.add_argument('--test-pytest', action='store_true', + default=environ.get('PYBUILD_TEST_PYTEST') == '1', + help='use pytest module in --test step') + tests.add_argument('--test-tox', action='store_true', + default=environ.get('PYBUILD_TEST_TOX') == '1', + help='use tox in --test step') + tests.add_argument('--test-custom', action='store_true', + default=environ.get('PYBUILD_TEST_CUSTOM') == '1', + help='use custom command in --test step') + + dirs = parser.add_argument_group('DIRECTORIES') + dirs.add_argument('-d', '--dir', action='store', metavar='DIR', + default=environ.get('PYBUILD_DIR', getcwd()), + help='source files directory - base for other relative dirs [default: CWD]') + dirs.add_argument('--dest-dir', action='store', metavar='DIR', dest='destdir', + default=environ.get('DESTDIR', 'debian/tmp'), + help='destination directory [default: debian/tmp]') + dirs.add_argument('--ext-dest-dir', action='store', metavar='DIR', dest='ext_destdir', + default=environ.get('PYBUILD_EXT_DESTDIR'), + help='destination directory for .so files') + dirs.add_argument('--ext-pattern', action='store', metavar='PATTERN', + default=environ.get('PYBUILD_EXT_PATTERN', r'\.so(\.[^/]*)?$'), + help='regular expression for files that should be moved' + ' if --ext-dest-dir is set [default: .so files]') + dirs.add_argument('--ext-sub-pattern', action='store', metavar='PATTERN', + default=environ.get('PYBUILD_EXT_SUB_PATTERN'), + help='pattern to change --ext-pattern\'s filename or path') + dirs.add_argument('--ext-sub-repl', action='store', metavar='PATTERN', + default=environ.get('PYBUILD_EXT_SUB_REPL'), + help='replacement for match from --ext-sub-pattern,' + ' empty string by default') + dirs.add_argument('--install-dir', action='store', metavar='DIR', + help='installation directory [default: .../dist-packages]') + dirs.add_argument('--name', action='store', + default=environ.get('PYBUILD_NAME'), + help='use this name to guess destination directories') + + limit = parser.add_argument_group('LIMITATIONS') + limit.add_argument('-s', '--system', + default=environ.get('PYBUILD_SYSTEM'), + help='select a build system [default: auto-detection]') + limit.add_argument('-p', '--pyver', action='append', dest='versions', + help='''build for Python VERSION. + This option can be used multiple times + [default: all supported Python 3.X versions]''') + limit.add_argument('-i', '--interpreter', action='append', + help='change interpreter [default: python{version}]') + limit.add_argument('--disable', metavar='ITEMS', + help='disable action, interpreter or version') + + args = parser.parse_args() + if not args.interpreter: + args.interpreter = environ.get('PYBUILD_INTERPRETERS', 'python{version}').split() + if not args.versions: + args.versions = environ.get('PYBUILD_VERSIONS', '').split() + else: + # add support for -p `pyversions -rv` + versions = [] + for version in args.versions: + versions.extend(version.split()) + args.versions = versions + + if args.test_nose or args.test_nose2 or args.test_pytest or args.test_tox\ + or args.test_custom or args.system == 'custom': + args.custom_tests = True + else: + args.custom_tests = False + + return args + + +if __name__ == '__main__': + cfg = parse_args(sys.argv) + if cfg.really_quiet: + cfg.quiet = True + log.setLevel(logging.CRITICAL) + elif cfg.verbose: + log.setLevel(logging.DEBUG) + else: + log.setLevel(logging.INFO) + log.debug('version: DEVELV') + log.debug(sys.argv) + main(cfg) + # let dh/cdbs clean the .pybuild dir + # rmtree(join(cfg.dir, '.pybuild')) |