#! /usr/bin/python3 # vim: et ts=4 sw=4 # Copyright © 2012-2013 Piotr Ożarowski # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. import logging import argparse import re import sys from os import environ, getcwd, makedirs, remove from os.path import abspath, exists, isdir, join from shutil import rmtree from tempfile import mkdtemp INTERP_VERSION_RE = re.compile(r'^python(?P3\.\d+)(?P-dbg)?$') logging.basicConfig(format='%(levelname).1s: pybuild ' '%(module)s:%(lineno)d: %(message)s') log = logging.getLogger('dhpython') def main(cfg): log.debug('cfg: %s', cfg) from dhpython import build, PKG_PREFIX_MAP from dhpython.debhelper import DebHelper, build_options from dhpython.version import Version, build_sorted, get_requested_versions from dhpython.interpreter import Interpreter from dhpython.tools import execute, move_matching_files if cfg.list_systems: for name, Plugin in sorted(build.plugins.items()): print(name, '\t', Plugin.DESCRIPTION) exit(0) nocheck = False if 'DEB_BUILD_OPTIONS' in environ: nocheck = 'nocheck' in environ['DEB_BUILD_OPTIONS'] if not nocheck and 'DEB_BUILD_PROFILES' in environ: nocheck = 'nocheck' in environ['DEB_BUILD_PROFILES'] env = environ.copy() # set some defaults in environ to make the build reproducible env.setdefault('LC_ALL', 'C.UTF-8') env.setdefault('CCACHE_DIR', abspath('.pybuild/ccache')) env.setdefault('no_proxy', 'localhost') if 'http_proxy' not in env: env['http_proxy'] = 'http://127.0.0.1:9/' elif not env['http_proxy']: del env['http_proxy'] # some tools don't like empty var. if 'https_proxy' not in env: env['https_proxy'] = 'https://127.0.0.1:9/' elif not env['https_proxy']: del env['https_proxy'] # some tools don't like empty var. if 'DEB_PYTHON_INSTALL_LAYOUT' not in env: env['DEB_PYTHON_INSTALL_LAYOUT'] = 'deb' arch_data = {} if exists('/usr/bin/dpkg-architecture'): res = execute('/usr/bin/dpkg-architecture') for line in res['stdout'].splitlines(): key, value = line.strip().split('=', 1) arch_data[key] = value # Set _PYTHON_HOST_PLATFORM to ensure debugging symbols on, f.e. i386 # emded a constant name regardless of the 32/64-bit kernel. host_platform = '{DEB_HOST_ARCH_OS}-{DEB_HOST_ARCH}'.format(**arch_data) # it's not called amd64 in Python host_platform = host_platform.replace('amd64', 'x86_64') env.setdefault('_PYTHON_HOST_PLATFORM', host_platform) if arch_data['DEB_BUILD_ARCH'] != arch_data['DEB_HOST_ARCH']: # support cross compiling Python 3.X extensions, see #892931 env.setdefault('_PYTHON_SYSCONFIGDATA_NAME', '_sysconfigdata__' + arch_data["DEB_HOST_MULTIARCH"]) # Selected on command line? selected_plugin = cfg.system # Selected by build_dep? if not selected_plugin: dh = DebHelper(build_options()) for build_dep in dh.build_depends: if build_dep.startswith('pybuild-plugin-'): selected_plugin = build_dep.split('-', 2)[2] break if selected_plugin: certainty = 99 Plugin = build.plugins.get(selected_plugin) if not Plugin: log.error('unrecognized build system: %s', selected_plugin) exit(10) plugin = Plugin(cfg) context = {'ENV': env, 'args': {}, 'dir': cfg.dir} plugin.detect(context) else: plugin, certainty, context = None, 0, None for Plugin in build.plugins.values(): try: tmp_plugin = Plugin(cfg) except Exception as err: log.warn('cannot initialize %s plugin: %s', Plugin.NAME, err, exc_info=cfg.verbose) continue tmp_context = {'ENV': env, 'args': {}, 'dir': cfg.dir} tmp_certainty = tmp_plugin.detect(tmp_context) log.debug('Plugin %s: certainty %i', Plugin.NAME, tmp_certainty) if tmp_certainty and tmp_certainty > certainty: plugin, certainty, context = tmp_plugin, tmp_certainty, tmp_context del Plugin if not plugin: log.error('cannot detect build system, please use --system option' ' or set PYBUILD_SYSTEM env. variable') exit(11) if plugin.SUPPORTED_INTERPRETERS is not True: # if versioned interpreter was requested and selected plugin lists # versioned ones as supported: extend list of supported interpreters # with this interpreter tpls = {i for i in plugin.SUPPORTED_INTERPRETERS if '{version}' in i} if tpls: for ipreter in cfg.interpreter: m = INTERP_VERSION_RE.match(ipreter) if m: ver = m.group('version') updated = set(tpl.format(version=ver) for tpl in tpls) updated and plugin.SUPPORTED_INTERPRETERS.update(updated) for interpreter in cfg.interpreter: if plugin.SUPPORTED_INTERPRETERS is not True and interpreter not in plugin.SUPPORTED_INTERPRETERS: log.error('interpreter %s not supported by %s', interpreter, plugin) exit(12) log.debug('detected build system: %s (certainty: %s%%)', plugin.NAME, certainty) if cfg.detect_only: if not cfg.really_quiet: print(plugin.NAME) exit(0) versions = cfg.versions if not versions: if len(cfg.interpreter) == 1: i = cfg.interpreter[0] m = INTERP_VERSION_RE.match(i) if m: log.debug('defaulting to version hardcoded in interpreter name') versions = [m.group('version')] else: IMAP = {v: k for k, v in PKG_PREFIX_MAP.items()} if i in IMAP: versions = build_sorted(get_requested_versions( IMAP[i], available=True), impl=IMAP[i]) if versions and '{version}' not in i: versions = versions[-1:] # last one, the default one if not versions: # still no luck log.debug('defaulting to all supported Python 3.X versions') versions = build_sorted(get_requested_versions( 'cpython3', available=True), impl='cpython3') versions = [Version(v) for v in versions] def get_option(name, interpreter=None, version=None, default=None): if interpreter: # try PYBUILD_NAME_python3.3-dbg (or hardcoded interpreter) i = interpreter.format(version=version or '') opt = "PYBUILD_{}_{}".format(name.upper(), i) if opt in environ: return environ[opt] # try PYBUILD_NAME_python3-dbg (if not checked above) if '{version}' in interpreter and version: i = interpreter.format(version=version.major) opt = "PYBUILD_{}_{}".format(name.upper(), i) if opt in environ: return environ[opt] # try PYBUILD_NAME opt = "PYBUILD_{}".format(name.upper()) if opt in environ: return environ[opt] # try command line args return getattr(cfg, name, default) or default def get_args(context, step, version, interpreter): i = interpreter.format(version=version) ipreter = Interpreter(i) home_dir = [ipreter.impl, str(version)] if ipreter.debug: home_dir.append('dbg') if cfg.name: home_dir.append(cfg.name) if cfg.autopkgtest_only: base_dir = environ.get('AUTOPKGTEST_TMP') if not base_dir: base_dir = mkdtemp(prefix='pybuild-autopkgtest-') else: base_dir = '.pybuild/{}' home_dir = base_dir.format('_'.join(home_dir)) build_dir = get_option('build_dir', interpreter, version, default=join(home_dir, 'build')) destdir = context['destdir'].format(version=version, interpreter=i) if cfg.name: package = ipreter.suggest_pkg_name(cfg.name) else: package = 'PYBUILD_NAME_not_set' if cfg.name and destdir.rstrip('/').endswith('debian/tmp'): destdir = "debian/{}".format(package) destdir = abspath(destdir) args = dict(context['args']) args.update({ 'package': package, 'interpreter': ipreter, 'version': version, 'args': get_option("%s_args" % step, interpreter, version, ''), 'dir': abspath(context['dir'].format(version=version, interpreter=i)), 'destdir': destdir, 'build_dir': abspath(build_dir.format(version=version, interpreter=i)), # versioned dist-packages even for Python 3.X - dh_python3 will fix it later # (and will have a chance to compare files) 'install_dir': get_option('install_dir', interpreter, version, '/usr/lib/python{version}/dist-packages' ).format(version=version, interpreter=i), 'home_dir': abspath(home_dir)}) if interpreter == 'pypy': args['install_dir'] = '/usr/lib/pypy/dist-packages/' env = dict(args.get('ENV', {})) pp = env.get('PYTHONPATH', context['ENV'].get('PYTHONPATH')) pp = pp.split(':') if pp else [] if step in {'build', 'test', 'autopkgtest'}: if step in {'test', 'autopkgtest'}: args['test_dir'] = join(args['destdir'], args['install_dir'].lstrip('/')) if args['test_dir'] not in pp: pp.append(args['test_dir']) if args['build_dir'] not in pp: pp.append(args['build_dir']) # cross compilation support for Python 2.x if (version.major == 2 and arch_data.get('DEB_BUILD_ARCH') != arch_data.get('DEB_HOST_ARCH')): pp.insert(0, ('/usr/lib/python{0}/plat-{1[DEB_HOST_MULTIARCH]}' ).format(version, arch_data)) env['PYTHONPATH'] = ':'.join(pp) # cross compilation support for Python <= 3.8 (see above) if version.major == 3: name = '_PYTHON_SYSCONFIGDATA_NAME' value = env.get(name, context['ENV'].get(name, '')) if version << '3.8' and value.startswith('_sysconfigdata_')\ and not value.startswith('_sysconfigdata_m'): value = env[name] = "_sysconfigdata_m%s" % value[15:] # update default from main() for -dbg interpreter if value and ipreter.debug and not value.startswith('_sysconfigdata_d'): env[name] = "_sysconfigdata_d%s" % value[15:] args['ENV'] = env if not exists(args['build_dir']): makedirs(args['build_dir']) return args def is_disabled(step, interpreter, version): i = interpreter prefix = "{}/".format(step) disabled = (get_option('disable', i, version) or '').split() for item in disabled: if item in (step, '1'): log.debug('disabling {} step for {} {}'.format(step, i, version)) return True if item.startswith(prefix): disabled.append(item[len(prefix):]) if i in disabled or str(version) in disabled or \ i.format(version=version) in disabled or \ i.format(version=version.major) in disabled: log.debug('disabling {} step for {} {}'.format(step, i, version)) return True return False def run(func, interpreter, version, context): step = func.__func__.__name__ args = get_args(context, step, version, interpreter) env = dict(context['ENV']) if 'ENV' in args: env.update(args['ENV']) before_cmd = get_option('before_{}'.format(step), interpreter, version) if before_cmd: if cfg.quiet: log_file = join(args['home_dir'], 'before_{}_cmd.log'.format(step)) else: log_file = False command = before_cmd.format(**args) log.info(command) output = execute(command, context['dir'], env, log_file) if output['returncode'] != 0: msg = 'exit code={}: {}'.format(output['returncode'], command) raise Exception(msg) fpath = join(args['home_dir'], 'testfiles_to_rm_before_install') if step == 'install' and exists(fpath): with open(fpath) as fp: for line in fp: path = line.strip('\n') if exists(path): if isdir(path): rmtree(path) else: remove(path) remove(fpath) result = func(context, args) after_cmd = get_option('after_{}'.format(step), interpreter, version) if after_cmd: if cfg.quiet: log_file = join(args['home_dir'], 'after_{}_cmd.log'.format(step)) else: log_file = False command = after_cmd.format(**args) log.info(command) output = execute(command, context['dir'], env, log_file) if output['returncode'] != 0: msg = 'exit code={}: {}'.format(output['returncode'], command) raise Exception(msg) return result def move_to_ext_destdir(i, version, context): """Move built C extensions from the general destdir to ext_destdir""" args = get_args(context, 'install', version, interpreter) ext_destdir = get_option('ext_destdir', i, version) if ext_destdir: move_matching_files(args['destdir'], ext_destdir, get_option('ext_pattern', i, version), get_option('ext_sub_pattern', i, version), get_option('ext_sub_repl', i, version)) func = None if cfg.clean_only: func = plugin.clean elif cfg.configure_only: func = plugin.configure elif cfg.build_only: func = plugin.build elif cfg.install_only: func = plugin.install elif cfg.test_only: func = plugin.test elif cfg.autopkgtest_only: func = plugin.test elif cfg.print_args: func = plugin.print_args ### one function for each interpreter at a time mode ### if func: step = func.__func__.__name__ if step == 'test' and nocheck: exit(0) failure = False for i in cfg.interpreter: ipreter = Interpreter(interpreter.format(version=versions[0])) iversions = build_sorted(versions, impl=ipreter.impl) if '{version}' not in i and len(versions) > 1: log.info('limiting Python versions to %s due to missing {version}' ' in interpreter string', str(versions[-1])) iversions = versions[-1:] # just the default or closest to default for version in iversions: if is_disabled(step, i, version): continue c = dict(context) c['dir'] = get_option('dir', i, version, cfg.dir) c['destdir'] = get_option('destdir', i, version, cfg.destdir) try: run(func, i, version, c) except Exception as err: log.error('%s: plugin %s failed with: %s', step, plugin.NAME, err, exc_info=cfg.verbose) # try to build/test other interpreters/versions even if # one of them fails to make build logs more verbose: failure = True if step not in ('build', 'test', 'autopkgtest'): exit(13) if step == 'install': move_to_ext_destdir(i, version, c) if failure: # exit with a non-zero return code if at least one build/test failed exit(13) exit(0) ### all functions for interpreters in batches mode ### try: context_map = {} for i in cfg.interpreter: ipreter = Interpreter(interpreter.format(version=versions[0])) iversions = build_sorted(versions, impl=ipreter.impl) if '{version}' not in i and len(versions) > 1: log.info('limiting Python versions to %s due to missing {version}' ' in interpreter string', str(versions[-1])) iversions = versions[-1:] # just the default or closest to default for version in iversions: key = (i, version) if key in context_map: c = context_map[key] else: c = dict(context) c['dir'] = get_option('dir', i, version, cfg.dir) c['destdir'] = get_option('destdir', i, version, cfg.destdir) context_map[key] = c if not is_disabled('clean', i, version): run(plugin.clean, i, version, c) if not is_disabled('configure', i, version): run(plugin.configure, i, version, c) if not is_disabled('build', i, version): run(plugin.build, i, version, c) if not is_disabled('install', i, version): run(plugin.install, i, version, c) move_to_ext_destdir(i, version, c) if not nocheck and not is_disabled('test', i, version): run(plugin.test, i, version, c) except Exception as err: log.error('plugin %s failed: %s', plugin.NAME, err, exc_info=cfg.verbose) exit(14) def parse_args(argv): usage = '%(prog)s [ACTION] [BUILD SYSTEM ARGS] [DIRECTORIES] [OPTIONS]' parser = argparse.ArgumentParser(usage=usage) parser.add_argument('-v', '--verbose', action='store_true', default=environ.get('PYBUILD_VERBOSE') == '1', help='turn verbose mode on') parser.add_argument('-q', '--quiet', action='store_true', default=environ.get('PYBUILD_QUIET') == '1', help='doesn\'t show external command\'s output') parser.add_argument('-qq', '--really-quiet', action='store_true', default=environ.get('PYBUILD_RQUIET') == '1', help='be quiet') parser.add_argument('--version', action='version', version='%(prog)s DEVELV') action = parser.add_argument_group('ACTION', '''The default is to build, install and test the library using detected build system version by version. Selecting one of following actions, will invoke given action for all versions - one by one - which (contrary to the default action) in some build systems can overwrite previous results.''') action.add_argument('--detect', action='store_true', dest='detect_only', help='return the name of detected build system') action.add_argument('--clean', action='store_true', dest='clean_only', help='clean files using auto-detected build system specific methods') action.add_argument('--configure', action='store_true', dest='configure_only', help='invoke configure step for all requested Python versions') action.add_argument('--build', action='store_true', dest='build_only', help='invoke build step for all requested Python versions') action.add_argument('--install', action='store_true', dest='install_only', help='invoke install step for all requested Python versions') action.add_argument('--test', action='store_true', dest='test_only', help='invoke tests for auto-detected build system') action.add_argument('--autopkgtest', action='store_true', dest='autopkgtest_only', help='invoke autopkgtests for auto-detected build system') action.add_argument('--list-systems', action='store_true', help='list available build systems and exit') action.add_argument('--print', action='append', dest='print_args', help="print pybuild's internal parameters") arguments = parser.add_argument_group('BUILD SYSTEM ARGS', ''' Additional arguments passed to the build system. --system=custom requires complete command.''') arguments.add_argument('--before-clean', metavar='CMD', help='invoked before the clean command') arguments.add_argument('--clean-args', metavar='ARGS') arguments.add_argument('--after-clean', metavar='CMD', help='invoked after the clean command') arguments.add_argument('--before-configure', metavar='CMD', help='invoked before the configure command') arguments.add_argument('--configure-args', metavar='ARGS') arguments.add_argument('--after-configure', metavar='CMD', help='invoked after the configure command') arguments.add_argument('--before-build', metavar='CMD', help='invoked before the build command') arguments.add_argument('--build-args', metavar='ARGS') arguments.add_argument('--after-build', metavar='CMD', help='invoked after the build command') arguments.add_argument('--before-install', metavar='CMD', help='invoked before the install command') arguments.add_argument('--install-args', metavar='ARGS') arguments.add_argument('--after-install', metavar='CMD', help='invoked after the install command') arguments.add_argument('--before-test', metavar='CMD', help='invoked before the test command') arguments.add_argument('--test-args', metavar='ARGS') arguments.add_argument('--after-test', metavar='CMD', help='invoked after the test command') tests = parser.add_argument_group('TESTS', '''\ unittest\'s discover is used by default (if available)''') tests.add_argument('--test-nose', action='store_true', default=environ.get('PYBUILD_TEST_NOSE') == '1', help='use nose module in --test step') tests.add_argument('--test-nose2', action='store_true', default=environ.get('PYBUILD_TEST_NOSE2') == '1', help='use nose2 module in --test step') tests.add_argument('--test-pytest', action='store_true', default=environ.get('PYBUILD_TEST_PYTEST') == '1', help='use pytest module in --test step') tests.add_argument('--test-tox', action='store_true', default=environ.get('PYBUILD_TEST_TOX') == '1', help='use tox in --test step') tests.add_argument('--test-custom', action='store_true', default=environ.get('PYBUILD_TEST_CUSTOM') == '1', help='use custom command in --test step') dirs = parser.add_argument_group('DIRECTORIES') dirs.add_argument('-d', '--dir', action='store', metavar='DIR', default=environ.get('PYBUILD_DIR', getcwd()), help='source files directory - base for other relative dirs [default: CWD]') dirs.add_argument('--dest-dir', action='store', metavar='DIR', dest='destdir', default=environ.get('DESTDIR', 'debian/tmp'), help='destination directory [default: debian/tmp]') dirs.add_argument('--ext-dest-dir', action='store', metavar='DIR', dest='ext_destdir', default=environ.get('PYBUILD_EXT_DESTDIR'), help='destination directory for .so files') dirs.add_argument('--ext-pattern', action='store', metavar='PATTERN', default=environ.get('PYBUILD_EXT_PATTERN', r'\.so(\.[^/]*)?$'), help='regular expression for files that should be moved' ' if --ext-dest-dir is set [default: .so files]') dirs.add_argument('--ext-sub-pattern', action='store', metavar='PATTERN', default=environ.get('PYBUILD_EXT_SUB_PATTERN'), help='pattern to change --ext-pattern\'s filename or path') dirs.add_argument('--ext-sub-repl', action='store', metavar='PATTERN', default=environ.get('PYBUILD_EXT_SUB_REPL'), help='replacement for match from --ext-sub-pattern,' ' empty string by default') dirs.add_argument('--install-dir', action='store', metavar='DIR', help='installation directory [default: .../dist-packages]') dirs.add_argument('--name', action='store', default=environ.get('PYBUILD_NAME'), help='use this name to guess destination directories') limit = parser.add_argument_group('LIMITATIONS') limit.add_argument('-s', '--system', default=environ.get('PYBUILD_SYSTEM'), help='select a build system [default: auto-detection]') limit.add_argument('-p', '--pyver', action='append', dest='versions', help='''build for Python VERSION. This option can be used multiple times [default: all supported Python 3.X versions]''') limit.add_argument('-i', '--interpreter', action='append', help='change interpreter [default: python{version}]') limit.add_argument('--disable', metavar='ITEMS', help='disable action, interpreter or version') args = parser.parse_args() if not args.interpreter: args.interpreter = environ.get('PYBUILD_INTERPRETERS', 'python{version}').split() if not args.versions: args.versions = environ.get('PYBUILD_VERSIONS', '').split() else: # add support for -p `pyversions -rv` versions = [] for version in args.versions: versions.extend(version.split()) args.versions = versions if args.test_nose or args.test_nose2 or args.test_pytest or args.test_tox\ or args.test_custom or args.system == 'custom': args.custom_tests = True else: args.custom_tests = False return args if __name__ == '__main__': cfg = parse_args(sys.argv) if cfg.really_quiet: cfg.quiet = True log.setLevel(logging.CRITICAL) elif cfg.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) log.debug('version: DEVELV') log.debug(sys.argv) main(cfg) # let dh/cdbs clean the .pybuild dir # rmtree(join(cfg.dir, '.pybuild'))