diff options
Diffstat (limited to 'dhpython/build')
| -rw-r--r-- | dhpython/build/__init__.py | 42 | ||||
| -rw-r--r-- | dhpython/build/base.py | 293 | ||||
| -rw-r--r-- | dhpython/build/plugin_autopkgtest.py | 38 | ||||
| -rw-r--r-- | dhpython/build/plugin_cmake.py | 71 | ||||
| -rw-r--r-- | dhpython/build/plugin_custom.py | 48 | ||||
| -rw-r--r-- | dhpython/build/plugin_distutils.py | 121 | ||||
| -rw-r--r-- | dhpython/build/plugin_flit.py | 170 | ||||
| -rw-r--r-- | dhpython/build/plugin_pyproject.py | 201 |
8 files changed, 984 insertions, 0 deletions
diff --git a/dhpython/build/__init__.py b/dhpython/build/__init__.py new file mode 100644 index 0000000..d350ccb --- /dev/null +++ b/dhpython/build/__init__.py @@ -0,0 +1,42 @@ +# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +from glob import glob1 +from os.path import dirname + +from dhpython.exceptions import RequiredCommandMissingException + +log = logging.getLogger('dhpython') + +plugins = {} +for i in sorted(i[7:-3] for i in glob1(dirname(__file__), 'plugin_*.py')): + try: + module = __import__("dhpython.build.plugin_%s" % i, fromlist=[i]) + module.BuildSystem.NAME = i + module.BuildSystem.is_usable() + plugins[i] = module.BuildSystem + except RequiredCommandMissingException as err: + log.debug("cannot initialize '%s' plugin: Missing command '%s'", i, err) + except Exception as err: + if log.level < logging.INFO: + log.debug("cannot initialize '%s' plugin", i, exc_info=True) + else: + log.debug("cannot initialize '%s' plugin: %s", i, err) diff --git a/dhpython/build/base.py b/dhpython/build/base.py new file mode 100644 index 0000000..427ef2e --- /dev/null +++ b/dhpython/build/base.py @@ -0,0 +1,293 @@ +# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +from functools import wraps +from glob import glob1 +from os import remove, walk +from os.path import exists, isdir, join +from subprocess import Popen, PIPE +from shutil import rmtree, copyfile, copytree +from dhpython.exceptions import RequiredCommandMissingException +from dhpython.tools import execute +try: + from shlex import quote +except ImportError: + # shlex.quote is new in Python 3.3 + def quote(s): + if not s: + return "''" + return "'" + s.replace("'", "'\"'\"'") + "'" + +log = logging.getLogger('dhpython') + + +def copy_test_files(dest='{build_dir}', + filelist='{home_dir}/testfiles_to_rm_before_install', + add_to_args=('test', 'tests')): + + def _copy_test_files(func): + + @wraps(func) + def __copy_test_files(self, context, args, *oargs, **kwargs): + files_to_copy = {'test', 'tests'} + # check debian/pybuild_pythonX.Y.testfiles + for tpl in ('_{i}{v}', '_{i}{m}', ''): + tpl = tpl.format(i=args['interpreter'].name, + v=args['version'], + m=args['version'].major) + fpath = join(args['dir'], 'debian/pybuild{}.testfiles'.format(tpl)) + if exists(fpath): + with open(fpath, encoding='utf-8') as fp: + # overwrite files_to_copy if .testfiles file found + files_to_copy = [line.strip() for line in fp.readlines() + if not line.startswith('#')] + break + + files_to_remove = set() + for name in files_to_copy: + src_dpath = join(args['dir'], name) + dst_dpath = join(dest.format(**args), name.rsplit('/', 1)[-1]) + if exists(src_dpath): + if not exists(dst_dpath): + if isdir(src_dpath): + copytree(src_dpath, dst_dpath) + else: + copyfile(src_dpath, dst_dpath) + files_to_remove.add(dst_dpath + '\n') + if not args['args'] and 'PYBUILD_TEST_ARGS' not in context['ENV']\ + and (self.cfg.test_pytest or self.cfg.test_nose) \ + and name in add_to_args: + args['args'] = name + if files_to_remove and filelist: + with open(filelist.format(**args), 'a') as fp: + fp.writelines(files_to_remove) + + return func(self, context, args, *oargs, **kwargs) + return __copy_test_files + return _copy_test_files + + +class Base: + """Base class for build system plugins + + :attr REQUIRED_COMMANDS: list of command checked by default in :meth:is_usable, + if one of them is missing, plugin cannot be used. + :type REQUIRED_COMMANDS: list of strings + :attr REQUIRED_FILES: list of files (or glob templates) required by given + build system + :attr OPTIONAL_FILES: dictionary of glob templates (key) and score (value) + used to detect if given plugin is the best one for the job + :type OPTIONAL_FILES: dict (key is a string, value is an int) + :attr SUPPORTED_INTERPRETERS: set of interpreter templates (with or without + {version}) supported by given plugin + """ + DESCRIPTION = '' + REQUIRED_COMMANDS = [] + REQUIRED_FILES = [] + OPTIONAL_FILES = {} + SUPPORTED_INTERPRETERS = {'python', 'python3', 'python-dbg', 'python3-dbg', + 'python{version}', 'python{version}-dbg'} + # files and directories to remove during clean step (other than .pyc): + CLEAN_FILES = {'.pytest_cache', '.coverage'} + + def __init__(self, cfg): + self.cfg = cfg + + def __repr__(self): + return "BuildSystem(%s)" % self.NAME + + @classmethod + def is_usable(cls): + for command in cls.REQUIRED_COMMANDS: + process = Popen(['which', command], stdout=PIPE, stderr=PIPE) + out, err = process.communicate() + if process.returncode != 0: + raise RequiredCommandMissingException(command) + + def detect(self, context): + """Return certainty level that this plugin describes the right build system + + This method is using cls.{REQUIRED,OPTIONAL}_FILES only by default, + please extend it in the plugin if more sofisticated methods can be used + for given build system. + + :return: 0 <= certainty <= 100 + :rtype: int + """ + result = 0 + + required_files_num = 0 + self.DETECTED_REQUIRED_FILES = {} # can be used in the plugin later + for tpl in self.REQUIRED_FILES: + found = False + for ftpl in tpl.split('|'): + res = glob1(context['dir'], ftpl) + if res: + found = True + self.DETECTED_REQUIRED_FILES.setdefault(tpl, []).extend(res) + if found: + required_files_num += 1 + # add max 50 points depending on how many required files are available + if self.REQUIRED_FILES: + result += int(required_files_num / len(self.REQUIRED_FILES) * 50) + + self.DETECTED_OPTIONAL_FILES = {} + for ftpl, score in self.OPTIONAL_FILES.items(): + res = glob1(context['dir'], ftpl) + if res: + result += score + self.DETECTED_OPTIONAL_FILES.setdefault(ftpl, []).extend(res) + if result > 100: + return 100 + return result + + def clean(self, context, args): + if self.cfg.test_tox: + tox_dir = join(args['dir'], '.tox') + if isdir(tox_dir): + try: + rmtree(tox_dir) + except Exception: + log.debug('cannot remove %s', tox_dir) + + for fn in self.CLEAN_FILES: + path = join(context['dir'], fn) + if isdir(path): + try: + rmtree(path) + except Exception: + log.debug('cannot remove %s', path) + elif exists(path): + try: + remove(path) + except Exception: + log.debug('cannot remove %s', path) + + for root, dirs, file_names in walk(context['dir']): + for name in dirs: + if name == '__pycache__': + dpath = join(root, name) + log.debug('removing dir: %s', dpath) + try: + rmtree(dpath) + except Exception: + log.debug('cannot remove %s', dpath) + else: + dirs.remove(name) + for fn in file_names: + if fn.endswith(('.pyc', '.pyo')): + fpath = join(root, fn) + log.debug('removing: %s', fpath) + try: + remove(fpath) + except Exception: + log.debug('cannot remove %s', fpath) + + def configure(self, context, args): + raise NotImplementedError("configure method not implemented in %s" % self.NAME) + + def install(self, context, args): + raise NotImplementedError("install method not implemented in %s" % self.NAME) + + def build(self, context, args): + raise NotImplementedError("build method not implemented in %s" % self.NAME) + + @copy_test_files() + def test(self, context, args): + if self.cfg.test_nose2: + return 'cd {build_dir}; {interpreter} -m nose2 -v {args}' + elif self.cfg.test_nose: + return 'cd {build_dir}; {interpreter} -m nose -v {args}' + elif self.cfg.test_pytest: + return 'cd {build_dir}; {interpreter} -m pytest {args}' + elif self.cfg.test_tox: + # tox will call pip to install the module. Let it install the + # module inside the virtualenv + pydistutils_cfg = join(args['home_dir'], '.pydistutils.cfg') + if exists(pydistutils_cfg): + remove(pydistutils_cfg) + return 'cd {build_dir}; tox -c {dir}/tox.ini --sitepackages -e py{version.major}{version.minor} {args}' + elif self.cfg.test_custom: + return 'cd {build_dir}; {args}' + elif args['version'] == '2.7' or args['version'] >> '3.1' or args['interpreter'] == 'pypy': + return 'cd {build_dir}; {interpreter} -m unittest discover -v {args}' + + def execute(self, context, args, command, log_file=None): + if log_file is False and self.cfg.really_quiet: + log_file = None + command = command.format(**args) + env = dict(context['ENV']) + if 'ENV' in args: + env.update(args['ENV']) + log.info(command) + return execute(command, context['dir'], env, log_file) + + def print_args(self, context, args): + cfg = self.cfg + if len(cfg.print_args) == 1 and len(cfg.interpreter) == 1 and '{version}' not in cfg.interpreter[0]: + i = cfg.print_args[0] + if '{' in i: + print(i.format(**args)) + else: + print(args.get(i, '')) + else: + for i in cfg.print_args: + if '{' in i: + print(i.format(**args)) + else: + print('{} {}: {}'.format(args['interpreter'], i, args.get(i, ''))) + + +def shell_command(func): + + @wraps(func) + def wrapped_func(self, context, args, *oargs, **kwargs): + command = kwargs.pop('command', None) + if not command: + command = func(self, context, args, *oargs, **kwargs) + if isinstance(command, int): # final result + return command + if not command: + log.warn('missing command ' + '(plugin=%s, method=%s, interpreter=%s, version=%s)', + self.NAME, func.__name__, + args.get('interpreter'), args.get('version')) + return command + + if self.cfg.quiet: + log_file = join(args['home_dir'], '{}_cmd.log'.format(func.__name__)) + else: + log_file = False + + quoted_args = dict((k, quote(v)) if k in ('dir', 'destdir') + or k.endswith('_dir') else (k, v) + for k, v in args.items()) + command = command.format(**quoted_args) + + output = self.execute(context, args, command, log_file) + if output['returncode'] != 0: + msg = 'exit code={}: {}'.format(output['returncode'], command) + if log_file: + msg += '\nfull command log is available in {}'.format(log_file) + raise Exception(msg) + return True + + return wrapped_func diff --git a/dhpython/build/plugin_autopkgtest.py b/dhpython/build/plugin_autopkgtest.py new file mode 100644 index 0000000..b9a76e7 --- /dev/null +++ b/dhpython/build/plugin_autopkgtest.py @@ -0,0 +1,38 @@ +# vim: et ts=4 sw=4 +# Copyright © 2021 Antonio Terceiro <terceiro@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from dhpython.build.base import Base, shell_command + + +class BuildSystem(Base): + """ + autopkgtest test runner. All the methods that are not related to + testing (configure, build, clean etc) are not implemented by this class, on + purpose. + """ + + NAME = "autopkgtest" + DESCRIPTION = 'autopkgtest test runner' + SUPPORTED_INTERPRETERS = {'python3', 'python{version}'} + + @shell_command + def test(self, context, args): + return super().test(context, args) diff --git a/dhpython/build/plugin_cmake.py b/dhpython/build/plugin_cmake.py new file mode 100644 index 0000000..7cf52e9 --- /dev/null +++ b/dhpython/build/plugin_cmake.py @@ -0,0 +1,71 @@ +# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from dhpython.build.base import Base, shell_command, copy_test_files + + +class BuildSystem(Base): + DESCRIPTION = 'CMake build system (using dh_auto_* commands)' + REQUIRED_COMMANDS = ['cmake'] + REQUIRED_FILES = ['CMakeLists.txt'] + OPTIONAL_FILES = {'cmake_uninstall.cmake': 10, 'CMakeCache.txt': 10} + + @shell_command + def clean(self, context, args): + super(BuildSystem, self).clean(context, args) + return 'dh_auto_clean --buildsystem=cmake' + + @shell_command + def configure(self, context, args): + return ('dh_auto_configure --buildsystem=cmake' + ' --builddirectory={build_dir} --' + # FindPythonInterp: + ' -DPYTHON_EXECUTABLE:FILEPATH=/usr/bin/{interpreter}' + ' -DPYTHON_LIBRARY:FILEPATH={interpreter.library_file}' + ' -DPYTHON_INCLUDE_DIR:PATH={interpreter.include_dir}' + # FindPython: + ' -DPython_EXECUTABLE=/usr/bin/{interpreter}' + ' -DPython_LIBRARY={interpreter.library_file}' + ' -DPython_INCLUDE_DIR={interpreter.include_dir}' + # FindPython3: + ' -DPython3_EXECUTABLE=/usr/bin/{interpreter}' + ' -DPython3_LIBRARY={interpreter.library_file}' + ' -DPython3_INCLUDE_DIR={interpreter.include_dir}' + ' {args}') + + @shell_command + def build(self, context, args): + return ('dh_auto_build --buildsystem=cmake' + ' --builddirectory={build_dir}' + ' -- {args}') + + @shell_command + def install(self, context, args): + return ('dh_auto_install --buildsystem=cmake' + ' --builddirectory={build_dir}' + ' --destdir={destdir}' + ' -- {args}') + + @shell_command + @copy_test_files() + def test(self, context, args): + return ('dh_auto_test --buildsystem=cmake' + ' --builddirectory={build_dir}' + ' -- {args}') diff --git a/dhpython/build/plugin_custom.py b/dhpython/build/plugin_custom.py new file mode 100644 index 0000000..8b317f2 --- /dev/null +++ b/dhpython/build/plugin_custom.py @@ -0,0 +1,48 @@ +# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from dhpython.build.base import Base, shell_command, copy_test_files + + +class BuildSystem(Base): + DESCRIPTION = 'use --*-args options to configure this system' + SUPPORTED_INTERPRETERS = True # all interpreters + + @shell_command + def clean(self, context, args): + super(BuildSystem, self).clean(context, args) + return args['args'] + + @shell_command + def configure(self, context, args): + return args['args'] + + @shell_command + def build(self, context, args): + return args['args'] + + @shell_command + def install(self, context, args): + return args['args'] + + @shell_command + @copy_test_files() + def test(self, context, args): + return args['args'] or super(BuildSystem, self).test(context, args) diff --git a/dhpython/build/plugin_distutils.py b/dhpython/build/plugin_distutils.py new file mode 100644 index 0000000..342ec6f --- /dev/null +++ b/dhpython/build/plugin_distutils.py @@ -0,0 +1,121 @@ +# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +from glob import glob1 +from os import remove +from os.path import exists, isdir, join +from shutil import rmtree +from dhpython.build.base import Base, shell_command, copy_test_files + +log = logging.getLogger('dhpython') +_setup_tpl = 'setup.py|setup-3.py' + + +def create_pydistutils_cfg(func): + """distutils doesn't have sane command-line API - this decorator creates + .pydistutils.cfg file to workaround it + + hint: if you think this is plain stupid, please don't read + distutils/setuptools/distribute sources + """ + + def wrapped_func(self, context, args, *oargs, **kwargs): + fpath = join(args['home_dir'], '.pydistutils.cfg') + if not exists(fpath): + with open(fpath, 'w', encoding='utf-8') as fp: + lines = ['[clean]\n', + 'all=1\n', + '[build]\n', + 'build_lib={}\n'.format(args['build_dir']), + '[install]\n', + 'force=1\n', + 'install_layout=deb\n', + 'install_scripts=$base/bin\n', + 'install_lib={}\n'.format(args['install_dir']), + 'prefix=/usr\n'] + log.debug('pydistutils config file:\n%s', ''.join(lines)) + fp.writelines(lines) + context['ENV']['HOME'] = args['home_dir'] + return func(self, context, args, *oargs, **kwargs) + + wrapped_func.__name__ = func.__name__ + return wrapped_func + + +class BuildSystem(Base): + DESCRIPTION = 'Distutils build system' + SUPPORTED_INTERPRETERS = {'python', 'python3', 'python{version}', + 'python-dbg', 'python3-dbg', 'python{version}-dbg', + 'pypy'} + REQUIRED_FILES = [_setup_tpl] + OPTIONAL_FILES = {'setup.cfg': 1, + 'requirements.txt': 1, + 'PKG-INFO': 10, + '*.egg-info': 10} + CLEAN_FILES = Base.CLEAN_FILES | {'build'} + + def detect(self, context): + result = super(BuildSystem, self).detect(context) + if _setup_tpl in self.DETECTED_REQUIRED_FILES: + context['args']['setup_py'] = self.DETECTED_REQUIRED_FILES[_setup_tpl][0] + else: + context['args']['setup_py'] = 'setup.py' + return result + + @shell_command + @create_pydistutils_cfg + def clean(self, context, args): + super(BuildSystem, self).clean(context, args) + if exists(args['interpreter'].binary()): + return '{interpreter} {setup_py} clean {args}' + return 0 # no need to invoke anything + + @shell_command + @create_pydistutils_cfg + def configure(self, context, args): + return '{interpreter} {setup_py} config {args}' + + @shell_command + @create_pydistutils_cfg + def build(self, context, args): + return '{interpreter.binary_dv} {setup_py} build {args}' + + @shell_command + @create_pydistutils_cfg + def install(self, context, args): + # remove egg-info dirs from build_dir + for fname in glob1(args['build_dir'], '*.egg-info'): + fpath = join(args['build_dir'], fname) + rmtree(fpath) if isdir(fpath) else remove(fpath) + + return '{interpreter.binary_dv} {setup_py} install --root {destdir} {args}' + + @shell_command + @create_pydistutils_cfg + @copy_test_files() + def test(self, context, args): + if not self.cfg.custom_tests: + fpath = join(args['dir'], args['setup_py']) + with open(fpath, 'rb') as fp: + if fp.read().find(b'test_suite') > 0: + # TODO: is that enough to detect if test target is available? + return '{interpreter} {setup_py} test {args}' + return super(BuildSystem, self).test(context, args) diff --git a/dhpython/build/plugin_flit.py b/dhpython/build/plugin_flit.py new file mode 100644 index 0000000..004d657 --- /dev/null +++ b/dhpython/build/plugin_flit.py @@ -0,0 +1,170 @@ +# Copyright © 2012-2020 Piotr Ożarowski <piotr@debian.org> +# © 2020 Scott Kitterman <scott@kitterman.com> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from fnmatch import fnmatch +from pathlib import Path +import copy +import csv +import logging +import os +import os.path as osp +import shutil +import sysconfig +try: + import tomli +except ModuleNotFoundError: + # Plugin still works, only needed for autodetection + pass +try: + from flit.install import Installer +except ImportError: + Installer = object + +from dhpython.build.base import Base, shell_command + +log = logging.getLogger('dhpython') + + +class DebianInstaller(Installer): + def install_directly(self, destdir, installdir): + """Install a module/package into package directory, and create its + scripts. + """ + if installdir[:1] == os.sep: + installdir = installdir[1:] + + vars_ = copy.copy(sysconfig.get_config_vars()) + vars_['base'] = destdir + vars_['base'] + try: + dirs = sysconfig.get_paths(scheme='deb_system', vars=vars_) + except KeyError: + # Debian hasn't patched sysconfig schemes until 3.10 + # TODO: Introduce a version check once sysconfig is patched. + dirs = sysconfig.get_paths(scheme='posix_prefix', vars=vars_) + + dirs['purelib'] = dirs['platlib'] = osp.join(destdir, installdir) + os.makedirs(dirs['purelib'], exist_ok=True) + os.makedirs(dirs['scripts'], exist_ok=True) + + dst = osp.join(dirs['purelib'], osp.basename(self.module.path)) + if osp.lexists(dst): + if osp.isdir(dst) and not osp.islink(dst): + shutil.rmtree(dst) + else: + os.unlink(dst) + + src = str(self.module.path) + if self.module.is_package: + log.info("Installing package %s -> %s", src, dst) + shutil.copytree(src, dst) + self._record_installed_directory(dst) + else: + log.info("Installing file %s -> %s", src, dst) + shutil.copy2(src, dst) + self.installed_files.append(dst) + + scripts = self.ini_info.entrypoints.get('console_scripts', {}) + if scripts: + log.info("Installing scripts to %s", dirs['scripts']) + self.install_scripts(scripts, dirs['scripts']) + + log.info("Writing dist-info %s", dirs['purelib']) + self.write_dist_info(dirs['purelib']) + # Remove direct_url.json - contents are not useful or reproduceable + for path in Path(dirs['purelib']).glob("*.dist-info/direct_url.json"): + path.unlink() + # Remove build path from RECORD files + for path in Path(dirs['purelib']).glob("*.dist-info/RECORD"): + with open(path) as f: + reader = csv.reader(f) + records = list(reader) + with open(path, 'w') as f: + writer = csv.writer(f) + for path, hash_, size in records: + path = path.replace(destdir, '') + if fnmatch(path, "*.dist-info/direct_url.json"): + continue + writer.writerow([path, hash_, size]) + + +class BuildSystem(Base): + DESCRIPTION = 'Flit build system' + SUPPORTED_INTERPRETERS = {'python3', 'python{version}'} + REQUIRED_FILES = ['pyproject.toml'] + OPTIONAL_FILES = {} + CLEAN_FILES = Base.CLEAN_FILES | {'build'} + + def detect(self, context): + """Return certainty level that this plugin describes the right build + system + + This method uses cls.{REQUIRED}_FILES (pyroject.toml) as well as + checking to see if build-backend is set to flit_core.buildapi. + + Score is 85 if both are present (to allow manually setting distutils to + score higher if set). + + :return: 0 <= certainty <= 100 + :rtype: int + """ + if Installer is object: + return 0 + + result = super().detect(context) + if result > 100: + return 100 + return result + + def clean(self, context, args): + super().clean(context, args) + if osp.exists(args['interpreter'].binary()): + log.debug("removing '%s' (and everything under it)", + args['build_dir']) + osp.isdir(args['build_dir']) and shutil.rmtree(args['build_dir']) + return 0 # no need to invoke anything + + def configure(self, context, args): + # Flit does not support binary extensions + return 0 # Not needed for flit + + def build(self, context, args): + log.warning("The pybuild flit plugin is deprecated, " + "please use the pyproject plugin instead.") + my_dir = Path(args['dir']) + install_kwargs = {'user': False, 'symlink': False, 'deps': 'none'} + DebianInstaller.from_ini_path(my_dir / 'pyproject.toml', + **install_kwargs).install_directly( + args['build_dir'], '') + # These get byte compiled too, although it's not logged. + return 0 # Not needed for flit + + def install(self, context, args): + my_dir = Path(args['dir']) + install_kwargs = {'user': False, 'symlink': False, 'deps': 'none'} + DebianInstaller.from_ini_path(my_dir / 'pyproject.toml', + **install_kwargs).install_directly( + args['destdir'], + args['install_dir']) + return 0 # Not needed for flit' + + @shell_command + def test(self, context, args): + return super().test(context, args) diff --git a/dhpython/build/plugin_pyproject.py b/dhpython/build/plugin_pyproject.py new file mode 100644 index 0000000..b24aa9a --- /dev/null +++ b/dhpython/build/plugin_pyproject.py @@ -0,0 +1,201 @@ +# Copyright © 2012-2020 Piotr Ożarowski <piotr@debian.org> +# © 2020 Scott Kitterman <scott@kitterman.com> +# © 2021 Stuart Prescott <stuart@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from pathlib import Path +import logging +import os.path as osp +import shutil +import sysconfig +try: + import tomli +except ModuleNotFoundError: + # Plugin still works, only needed for autodetection + pass +try: + from installer import install + from installer.destinations import SchemeDictionaryDestination + from installer.sources import WheelFile +except ModuleNotFoundError: + SchemeDictionaryDestination = WheelFile = install = None + +from dhpython.build.base import Base, shell_command + +log = logging.getLogger('dhpython') + + +class BuildSystem(Base): + DESCRIPTION = 'Generic PEP517 build system' + SUPPORTED_INTERPRETERS = {'python3', 'python{version}'} + REQUIRED_FILES = ['pyproject.toml'] + OPTIONAL_FILES = {} + CLEAN_FILES = Base.CLEAN_FILES | {'build'} + + def detect(self, context): + """Return certainty level that this plugin describes the right build + system + + This method uses cls.{REQUIRED}_FILES (pyroject.toml) only; any + other PEP517 compliant builder (such as the flit) builder should + indicate higher specificity than this plugin. + + :return: 0 <= certainty <= 100 + :rtype: int + """ + result = super().detect(context) + # Temporarily reduce the threshold while we're in beta + result -= 20 + + try: + with open('pyproject.toml', 'rb') as f: + pyproject = tomli.load(f) + if pyproject.get('build-system', {}).get('build-backend'): + result += 10 + else: + # Not a PEP517 built package + result = 0 + except NameError: + # No toml, no autdetection + result = 0 + except FileNotFoundError: + # Not a PEP517 package + result = 0 + if result > 100: + return 100 + return result + + def clean(self, context, args): + super().clean(context, args) + if osp.exists(args['interpreter'].binary()): + log.debug("removing '%s' (and everything under it)", + args['build_dir']) + osp.isdir(args['build_dir']) and shutil.rmtree(args['build_dir']) + return 0 # no need to invoke anything + + def configure(self, context, args): + if install is None: + raise Exception("PEP517 plugin dependencies are not available. " + "Please Build-Depend on pybuild-plugin-pyproject.") + # No separate configure step + return 0 + + def build(self, context, args): + self.build_step1(context, args) + self.build_step2(context, args) + + @shell_command + def build_step1(self, context, args): + """ build a wheel using the PEP517 builder defined by upstream """ + log.info('Building wheel for %s with "build" module', + args['interpreter']) + context['ENV']['FLIT_NO_NETWORK'] = '1' + context['ENV']['HOME'] = args['home_dir'] + return ('{interpreter} -m build ' + '--skip-dependency-check --no-isolation --wheel ' + '--outdir ' + args['home_dir'] + + ' {args}' + ) + + def build_step2(self, context, args): + """ unpack the wheel into pybuild's normal """ + log.info('Unpacking wheel built for %s with "installer" module', + args['interpreter']) + extras = {} + for extra in ('scripts', 'data'): + path = Path(args["home_dir"]) / extra + if osp.exists(path): + log.warning(f'{extra.title()} directory already exists, ' + 'skipping unpack. ' + 'Is the Python package being built twice?') + return + extras[extra] = path + destination = SchemeDictionaryDestination( + { + 'platlib': args['build_dir'], + 'purelib': args['build_dir'], + 'scripts': extras['scripts'], + 'data': extras['data'], + }, + interpreter=args['interpreter'].binary_dv, + script_kind='posix', + ) + + # FIXME this next step will unpack every single wheel file it finds + # which is probably ok since each wheel is built in a separate + # directory; but perhaps it should only accept the correctly named + # wheels that match the current interpreter? + # python-packaging has relevant utilities in + # - packaging/utils.py::parse_wheel_filename + # - packaging/tags.py (although it is current-interpreter-centric) + wheels = Path(args['home_dir']).glob('*.whl') + for wheel in wheels: + if wheel.name.startswith('UNKNOWN'): + raise Exception(f'UNKNOWN wheel found: {wheel.name}. Does ' + 'pyproject.toml specify a build-backend?') + with WheelFile.open(wheel) as source: + install( + source=source, + destination=destination, + additional_metadata={}, + ) + + def install(self, context, args): + log.info('Copying package built for %s to destdir', + args['interpreter']) + try: + paths = sysconfig.get_paths(scheme='deb_system') + except KeyError: + # Debian hasn't patched sysconfig schemes until 3.10 + # TODO: Introduce a version check once sysconfig is patched. + paths = sysconfig.get_paths(scheme='posix_prefix') + + # start by copying the data and scripts + for extra in ('data', 'scripts'): + src_dir = Path(args['home_dir']) / extra + if not src_dir.exists(): + continue + target_dir = args['destdir'] + paths[extra] + log.debug('Copying %s directory contents from %s -> %s', + extra, src_dir, target_dir) + shutil.copytree( + src_dir, + target_dir, + dirs_exist_ok=True, + ) + + # then copy the modules + module_dir = args['build_dir'] + target_dir = args['destdir'] + args['install_dir'] + log.debug('Copying module contents from %s -> %s', + module_dir, target_dir) + shutil.copytree( + module_dir, + target_dir, + dirs_exist_ok=True, + ) + + @shell_command + def test(self, context, args): + scripts = Path(args["home_dir"]) / 'scripts' + if scripts.exists(): + context['ENV']['PATH'] = f"{scripts}:{context['ENV']['PATH']}" + context['ENV']['HOME'] = args['home_dir'] + return super().test(context, args) |