diff options
Diffstat (limited to 'dhpython')
| -rw-r--r-- | dhpython/__init__.py | 113 | ||||
| -rwxr-xr-x | dhpython/_defaults.py | 99 | ||||
| -rw-r--r-- | dhpython/build/__init__.py | 42 | ||||
| -rw-r--r-- | dhpython/build/base.py | 293 | ||||
| -rw-r--r-- | dhpython/build/plugin_autopkgtest.py | 38 | ||||
| -rw-r--r-- | dhpython/build/plugin_cmake.py | 71 | ||||
| -rw-r--r-- | dhpython/build/plugin_custom.py | 48 | ||||
| -rw-r--r-- | dhpython/build/plugin_distutils.py | 121 | ||||
| -rw-r--r-- | dhpython/build/plugin_flit.py | 170 | ||||
| -rw-r--r-- | dhpython/build/plugin_pyproject.py | 201 | ||||
| -rw-r--r-- | dhpython/debhelper.py | 327 | ||||
| -rw-r--r-- | dhpython/depends.py | 281 | ||||
| -rw-r--r-- | dhpython/exceptions.py | 23 | ||||
| -rw-r--r-- | dhpython/fs.py | 587 | ||||
| -rw-r--r-- | dhpython/interpreter.py | 576 | ||||
| -rw-r--r-- | dhpython/markers.py | 70 | ||||
| -rw-r--r-- | dhpython/option.py | 30 | ||||
| -rw-r--r-- | dhpython/pydist.py | 692 | ||||
| -rw-r--r-- | dhpython/tools.py | 340 | ||||
| -rw-r--r-- | dhpython/version.py | 457 |
20 files changed, 4579 insertions, 0 deletions
diff --git a/dhpython/__init__.py b/dhpython/__init__.py new file mode 100644 index 0000000..338eb17 --- /dev/null +++ b/dhpython/__init__.py @@ -0,0 +1,113 @@ +# Copyright © 2010-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import re + +PKG_PREFIX_MAP = {'cpython2': 'python', + 'cpython3': 'python3', + 'pypy': 'pypy'} + +# minimum version required for compile/clean scripts: +MINPYCDEP = {'cpython2': 'python2:any', + 'cpython3': 'python3:any', + 'pypy': 'pypy'} + +PUBLIC_DIR_RE = { + 'cpython2': re.compile(r'.*?/usr/lib/python(2\.\d)(?:/|$)'), + 'cpython3': re.compile(r'.*?/usr/lib/python(3(?:\.\d+)?)(?:/|$)'), + 'pypy': re.compile(r'.*?/usr/lib/pypy(?:/|$)')} + +INTERPRETER_DIR_TPLS = { + 'cpython2': r'.*/python2\.\d/', + 'cpython3': r'.*/python3(?:\.\d+)?/', + 'pypy': r'.*/pypy/'} + +MULTIARCH_DIR_TPL = re.compile( + '.*/([a-z][^/-]+-(?:linux|kfreebsd|gnu)(?:-[^/-]+)?)(?:/.*|$)') + +# Interpreter site-directories +OLD_SITE_DIRS = { + 'cpython2': [ + '/usr/local/lib/python{}/site-packages', + '/usr/local/lib/python{}/dist-packages', + '/var/lib/python-support/python{}', + '/usr/lib/pymodules/python{}', + lambda version: '/usr/lib/python{}/site-packages'.format(version) + if version >= '2.6' else None], + 'cpython3': [ + '/usr/local/lib/python{}/site-packages', + '/usr/local/lib/python{}/dist-packages', + '/usr/lib/python{}/site-packages', + '/usr/lib/python{}/dist-packages', + '/var/lib/python-support/python{}', + '/usr/lib/pymodules/python{}'], + 'pypy': [ + '/usr/local/lib/pypy/site-packages', + '/usr/local/lib/pypy/dist-packages', + '/usr/lib/pypy/site-packages']} + +# PyDist related +PYDIST_DIRS = { + 'cpython2': '/usr/share/python/dist/', + 'cpython3': '/usr/share/python3/dist/', + 'pypy': '/usr/share/pypy/dist/'} + +PYDIST_OVERRIDES_FNAMES = { + 'cpython2': 'debian/pydist-overrides', + 'cpython3': 'debian/py3dist-overrides', + 'pypy': 'debian/pypydist-overrides'} + +PYDIST_DPKG_SEARCH_TPLS = { + # implementation: (dpkg -S query, regex filter) + 'cpython2': ('*/{}-?*.*-info', + r'/(python2\..|pyshared)/.*.(egg|dist)-info$'), + 'cpython3': ('*python3/*/{}-?*.*-info', r'.(egg|dist)-info$'), + 'pypy': ('*/pypy/dist-packages/{}-?*.*-info', r'.(egg|dist)-info$'), +} + +# DebHelper related +DEPENDS_SUBSTVARS = { + 'cpython2': '${python:Depends}', + 'cpython3': '${python3:Depends}', + 'pypy': '${pypy:Depends}', +} +PKG_NAME_TPLS = { + 'cpython2': ('python-', 'python2.'), + 'cpython3': ('python3-', 'python3.'), + 'pypy': ('pypy-',) +} +RT_LOCATIONS = { + 'cpython2': '/usr/share/python/runtime.d/', + 'cpython3': '/usr/share/python3/runtime.d/', + 'pypy': '/usr/share/pypy/runtime.d/', +} +RT_TPLS = { + 'cpython2': ''' +if [ "$1" = rtupdate ]; then +\tpyclean {pkg_arg} {dname} +\tpycompile {pkg_arg} {args} {dname} +fi''', + 'cpython3': ''' +if [ "$1" = rtupdate ]; then +\tpy3clean {pkg_arg} {dname} +\tpy3compile {pkg_arg} {args} {dname} +fi''', + 'pypy': '' +} diff --git a/dhpython/_defaults.py b/dhpython/_defaults.py new file mode 100755 index 0000000..cf7537b --- /dev/null +++ b/dhpython/_defaults.py @@ -0,0 +1,99 @@ +#! /usr/bin/python3 +# Copyright © 2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +from configparser import ConfigParser +from os import environ +from os.path import exists +from subprocess import Popen, PIPE + +SUPPORTED = { + 'cpython2': [(2, 7)], + 'cpython3': [(3, 8)], + 'pypy': [(4, 0)]} +DEFAULT = { + 'cpython2': (2, 7), + 'cpython3': (3, 8), + 'pypy': (4, 0)} + +log = logging.getLogger('dhpython') + + +def cpython_versions(major): + result = [None, None] + ver = '' if major == 2 else '3' + supported = environ.get("DEBPYTHON{}_SUPPORTED".format(ver)) + default = environ.get("DEBPYTHON{}_DEFAULT".format(ver)) + if not supported or not default: + config = ConfigParser() + config.read("/usr/share/python{}/debian_defaults".format(ver)) + if not default: + default = config.get('DEFAULT', 'default-version', fallback='')[6:] + if not supported: + supported = config.get('DEFAULT', 'supported-versions', fallback='')\ + .replace('python', '') + if default: + try: + result[0] = tuple(int(i) for i in default.split('.')) + except Exception as err: + log.warn('invalid debian_defaults file: %s', err) + if supported: + try: + result[1] = tuple(tuple(int(j) for j in i.strip().split('.')) + for i in supported.split(',')) + except Exception as err: + log.warn('invalid debian_defaults file: %s', err) + return result + + +def from_file(fpath): + if not exists(fpath): + raise ValueError("missing interpreter: %s" % fpath) + command = "{} --version".format(fpath) + with Popen(command, shell=True, stdout=PIPE) as process: + stdout, stderr = process.communicate() + stdout = str(stdout, 'utf-8') + + print(stdout) + + +cpython2 = cpython_versions(2) +cpython3 = cpython_versions(3) +if cpython2[0]: + DEFAULT['cpython2'] = cpython2[0] +if cpython3[0]: + DEFAULT['cpython3'] = cpython3[0] +if cpython2[1]: + SUPPORTED['cpython2'] = cpython2[1] +if cpython3[1]: + SUPPORTED['cpython3'] = cpython3[1] +#from_file('/usr/bin/pypy') + + +if __name__ == '__main__': + from sys import argv, stderr + if len(argv) != 3: + print('invalid number of arguments', file=stderr) + exit(1) + if argv[1] == 'default': + print('.'.join(str(i) for i in DEFAULT[argv[2]])) + elif argv[1] == 'supported': + print(','.join(('.'.join(str(i) for i in v) for v in SUPPORTED[argv[2]]))) diff --git a/dhpython/build/__init__.py b/dhpython/build/__init__.py new file mode 100644 index 0000000..d350ccb --- /dev/null +++ b/dhpython/build/__init__.py @@ -0,0 +1,42 @@ +# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +from glob import glob1 +from os.path import dirname + +from dhpython.exceptions import RequiredCommandMissingException + +log = logging.getLogger('dhpython') + +plugins = {} +for i in sorted(i[7:-3] for i in glob1(dirname(__file__), 'plugin_*.py')): + try: + module = __import__("dhpython.build.plugin_%s" % i, fromlist=[i]) + module.BuildSystem.NAME = i + module.BuildSystem.is_usable() + plugins[i] = module.BuildSystem + except RequiredCommandMissingException as err: + log.debug("cannot initialize '%s' plugin: Missing command '%s'", i, err) + except Exception as err: + if log.level < logging.INFO: + log.debug("cannot initialize '%s' plugin", i, exc_info=True) + else: + log.debug("cannot initialize '%s' plugin: %s", i, err) diff --git a/dhpython/build/base.py b/dhpython/build/base.py new file mode 100644 index 0000000..427ef2e --- /dev/null +++ b/dhpython/build/base.py @@ -0,0 +1,293 @@ +# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +from functools import wraps +from glob import glob1 +from os import remove, walk +from os.path import exists, isdir, join +from subprocess import Popen, PIPE +from shutil import rmtree, copyfile, copytree +from dhpython.exceptions import RequiredCommandMissingException +from dhpython.tools import execute +try: + from shlex import quote +except ImportError: + # shlex.quote is new in Python 3.3 + def quote(s): + if not s: + return "''" + return "'" + s.replace("'", "'\"'\"'") + "'" + +log = logging.getLogger('dhpython') + + +def copy_test_files(dest='{build_dir}', + filelist='{home_dir}/testfiles_to_rm_before_install', + add_to_args=('test', 'tests')): + + def _copy_test_files(func): + + @wraps(func) + def __copy_test_files(self, context, args, *oargs, **kwargs): + files_to_copy = {'test', 'tests'} + # check debian/pybuild_pythonX.Y.testfiles + for tpl in ('_{i}{v}', '_{i}{m}', ''): + tpl = tpl.format(i=args['interpreter'].name, + v=args['version'], + m=args['version'].major) + fpath = join(args['dir'], 'debian/pybuild{}.testfiles'.format(tpl)) + if exists(fpath): + with open(fpath, encoding='utf-8') as fp: + # overwrite files_to_copy if .testfiles file found + files_to_copy = [line.strip() for line in fp.readlines() + if not line.startswith('#')] + break + + files_to_remove = set() + for name in files_to_copy: + src_dpath = join(args['dir'], name) + dst_dpath = join(dest.format(**args), name.rsplit('/', 1)[-1]) + if exists(src_dpath): + if not exists(dst_dpath): + if isdir(src_dpath): + copytree(src_dpath, dst_dpath) + else: + copyfile(src_dpath, dst_dpath) + files_to_remove.add(dst_dpath + '\n') + if not args['args'] and 'PYBUILD_TEST_ARGS' not in context['ENV']\ + and (self.cfg.test_pytest or self.cfg.test_nose) \ + and name in add_to_args: + args['args'] = name + if files_to_remove and filelist: + with open(filelist.format(**args), 'a') as fp: + fp.writelines(files_to_remove) + + return func(self, context, args, *oargs, **kwargs) + return __copy_test_files + return _copy_test_files + + +class Base: + """Base class for build system plugins + + :attr REQUIRED_COMMANDS: list of command checked by default in :meth:is_usable, + if one of them is missing, plugin cannot be used. + :type REQUIRED_COMMANDS: list of strings + :attr REQUIRED_FILES: list of files (or glob templates) required by given + build system + :attr OPTIONAL_FILES: dictionary of glob templates (key) and score (value) + used to detect if given plugin is the best one for the job + :type OPTIONAL_FILES: dict (key is a string, value is an int) + :attr SUPPORTED_INTERPRETERS: set of interpreter templates (with or without + {version}) supported by given plugin + """ + DESCRIPTION = '' + REQUIRED_COMMANDS = [] + REQUIRED_FILES = [] + OPTIONAL_FILES = {} + SUPPORTED_INTERPRETERS = {'python', 'python3', 'python-dbg', 'python3-dbg', + 'python{version}', 'python{version}-dbg'} + # files and directories to remove during clean step (other than .pyc): + CLEAN_FILES = {'.pytest_cache', '.coverage'} + + def __init__(self, cfg): + self.cfg = cfg + + def __repr__(self): + return "BuildSystem(%s)" % self.NAME + + @classmethod + def is_usable(cls): + for command in cls.REQUIRED_COMMANDS: + process = Popen(['which', command], stdout=PIPE, stderr=PIPE) + out, err = process.communicate() + if process.returncode != 0: + raise RequiredCommandMissingException(command) + + def detect(self, context): + """Return certainty level that this plugin describes the right build system + + This method is using cls.{REQUIRED,OPTIONAL}_FILES only by default, + please extend it in the plugin if more sofisticated methods can be used + for given build system. + + :return: 0 <= certainty <= 100 + :rtype: int + """ + result = 0 + + required_files_num = 0 + self.DETECTED_REQUIRED_FILES = {} # can be used in the plugin later + for tpl in self.REQUIRED_FILES: + found = False + for ftpl in tpl.split('|'): + res = glob1(context['dir'], ftpl) + if res: + found = True + self.DETECTED_REQUIRED_FILES.setdefault(tpl, []).extend(res) + if found: + required_files_num += 1 + # add max 50 points depending on how many required files are available + if self.REQUIRED_FILES: + result += int(required_files_num / len(self.REQUIRED_FILES) * 50) + + self.DETECTED_OPTIONAL_FILES = {} + for ftpl, score in self.OPTIONAL_FILES.items(): + res = glob1(context['dir'], ftpl) + if res: + result += score + self.DETECTED_OPTIONAL_FILES.setdefault(ftpl, []).extend(res) + if result > 100: + return 100 + return result + + def clean(self, context, args): + if self.cfg.test_tox: + tox_dir = join(args['dir'], '.tox') + if isdir(tox_dir): + try: + rmtree(tox_dir) + except Exception: + log.debug('cannot remove %s', tox_dir) + + for fn in self.CLEAN_FILES: + path = join(context['dir'], fn) + if isdir(path): + try: + rmtree(path) + except Exception: + log.debug('cannot remove %s', path) + elif exists(path): + try: + remove(path) + except Exception: + log.debug('cannot remove %s', path) + + for root, dirs, file_names in walk(context['dir']): + for name in dirs: + if name == '__pycache__': + dpath = join(root, name) + log.debug('removing dir: %s', dpath) + try: + rmtree(dpath) + except Exception: + log.debug('cannot remove %s', dpath) + else: + dirs.remove(name) + for fn in file_names: + if fn.endswith(('.pyc', '.pyo')): + fpath = join(root, fn) + log.debug('removing: %s', fpath) + try: + remove(fpath) + except Exception: + log.debug('cannot remove %s', fpath) + + def configure(self, context, args): + raise NotImplementedError("configure method not implemented in %s" % self.NAME) + + def install(self, context, args): + raise NotImplementedError("install method not implemented in %s" % self.NAME) + + def build(self, context, args): + raise NotImplementedError("build method not implemented in %s" % self.NAME) + + @copy_test_files() + def test(self, context, args): + if self.cfg.test_nose2: + return 'cd {build_dir}; {interpreter} -m nose2 -v {args}' + elif self.cfg.test_nose: + return 'cd {build_dir}; {interpreter} -m nose -v {args}' + elif self.cfg.test_pytest: + return 'cd {build_dir}; {interpreter} -m pytest {args}' + elif self.cfg.test_tox: + # tox will call pip to install the module. Let it install the + # module inside the virtualenv + pydistutils_cfg = join(args['home_dir'], '.pydistutils.cfg') + if exists(pydistutils_cfg): + remove(pydistutils_cfg) + return 'cd {build_dir}; tox -c {dir}/tox.ini --sitepackages -e py{version.major}{version.minor} {args}' + elif self.cfg.test_custom: + return 'cd {build_dir}; {args}' + elif args['version'] == '2.7' or args['version'] >> '3.1' or args['interpreter'] == 'pypy': + return 'cd {build_dir}; {interpreter} -m unittest discover -v {args}' + + def execute(self, context, args, command, log_file=None): + if log_file is False and self.cfg.really_quiet: + log_file = None + command = command.format(**args) + env = dict(context['ENV']) + if 'ENV' in args: + env.update(args['ENV']) + log.info(command) + return execute(command, context['dir'], env, log_file) + + def print_args(self, context, args): + cfg = self.cfg + if len(cfg.print_args) == 1 and len(cfg.interpreter) == 1 and '{version}' not in cfg.interpreter[0]: + i = cfg.print_args[0] + if '{' in i: + print(i.format(**args)) + else: + print(args.get(i, '')) + else: + for i in cfg.print_args: + if '{' in i: + print(i.format(**args)) + else: + print('{} {}: {}'.format(args['interpreter'], i, args.get(i, ''))) + + +def shell_command(func): + + @wraps(func) + def wrapped_func(self, context, args, *oargs, **kwargs): + command = kwargs.pop('command', None) + if not command: + command = func(self, context, args, *oargs, **kwargs) + if isinstance(command, int): # final result + return command + if not command: + log.warn('missing command ' + '(plugin=%s, method=%s, interpreter=%s, version=%s)', + self.NAME, func.__name__, + args.get('interpreter'), args.get('version')) + return command + + if self.cfg.quiet: + log_file = join(args['home_dir'], '{}_cmd.log'.format(func.__name__)) + else: + log_file = False + + quoted_args = dict((k, quote(v)) if k in ('dir', 'destdir') + or k.endswith('_dir') else (k, v) + for k, v in args.items()) + command = command.format(**quoted_args) + + output = self.execute(context, args, command, log_file) + if output['returncode'] != 0: + msg = 'exit code={}: {}'.format(output['returncode'], command) + if log_file: + msg += '\nfull command log is available in {}'.format(log_file) + raise Exception(msg) + return True + + return wrapped_func diff --git a/dhpython/build/plugin_autopkgtest.py b/dhpython/build/plugin_autopkgtest.py new file mode 100644 index 0000000..b9a76e7 --- /dev/null +++ b/dhpython/build/plugin_autopkgtest.py @@ -0,0 +1,38 @@ +# vim: et ts=4 sw=4 +# Copyright © 2021 Antonio Terceiro <terceiro@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from dhpython.build.base import Base, shell_command + + +class BuildSystem(Base): + """ + autopkgtest test runner. All the methods that are not related to + testing (configure, build, clean etc) are not implemented by this class, on + purpose. + """ + + NAME = "autopkgtest" + DESCRIPTION = 'autopkgtest test runner' + SUPPORTED_INTERPRETERS = {'python3', 'python{version}'} + + @shell_command + def test(self, context, args): + return super().test(context, args) diff --git a/dhpython/build/plugin_cmake.py b/dhpython/build/plugin_cmake.py new file mode 100644 index 0000000..7cf52e9 --- /dev/null +++ b/dhpython/build/plugin_cmake.py @@ -0,0 +1,71 @@ +# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from dhpython.build.base import Base, shell_command, copy_test_files + + +class BuildSystem(Base): + DESCRIPTION = 'CMake build system (using dh_auto_* commands)' + REQUIRED_COMMANDS = ['cmake'] + REQUIRED_FILES = ['CMakeLists.txt'] + OPTIONAL_FILES = {'cmake_uninstall.cmake': 10, 'CMakeCache.txt': 10} + + @shell_command + def clean(self, context, args): + super(BuildSystem, self).clean(context, args) + return 'dh_auto_clean --buildsystem=cmake' + + @shell_command + def configure(self, context, args): + return ('dh_auto_configure --buildsystem=cmake' + ' --builddirectory={build_dir} --' + # FindPythonInterp: + ' -DPYTHON_EXECUTABLE:FILEPATH=/usr/bin/{interpreter}' + ' -DPYTHON_LIBRARY:FILEPATH={interpreter.library_file}' + ' -DPYTHON_INCLUDE_DIR:PATH={interpreter.include_dir}' + # FindPython: + ' -DPython_EXECUTABLE=/usr/bin/{interpreter}' + ' -DPython_LIBRARY={interpreter.library_file}' + ' -DPython_INCLUDE_DIR={interpreter.include_dir}' + # FindPython3: + ' -DPython3_EXECUTABLE=/usr/bin/{interpreter}' + ' -DPython3_LIBRARY={interpreter.library_file}' + ' -DPython3_INCLUDE_DIR={interpreter.include_dir}' + ' {args}') + + @shell_command + def build(self, context, args): + return ('dh_auto_build --buildsystem=cmake' + ' --builddirectory={build_dir}' + ' -- {args}') + + @shell_command + def install(self, context, args): + return ('dh_auto_install --buildsystem=cmake' + ' --builddirectory={build_dir}' + ' --destdir={destdir}' + ' -- {args}') + + @shell_command + @copy_test_files() + def test(self, context, args): + return ('dh_auto_test --buildsystem=cmake' + ' --builddirectory={build_dir}' + ' -- {args}') diff --git a/dhpython/build/plugin_custom.py b/dhpython/build/plugin_custom.py new file mode 100644 index 0000000..8b317f2 --- /dev/null +++ b/dhpython/build/plugin_custom.py @@ -0,0 +1,48 @@ +# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from dhpython.build.base import Base, shell_command, copy_test_files + + +class BuildSystem(Base): + DESCRIPTION = 'use --*-args options to configure this system' + SUPPORTED_INTERPRETERS = True # all interpreters + + @shell_command + def clean(self, context, args): + super(BuildSystem, self).clean(context, args) + return args['args'] + + @shell_command + def configure(self, context, args): + return args['args'] + + @shell_command + def build(self, context, args): + return args['args'] + + @shell_command + def install(self, context, args): + return args['args'] + + @shell_command + @copy_test_files() + def test(self, context, args): + return args['args'] or super(BuildSystem, self).test(context, args) diff --git a/dhpython/build/plugin_distutils.py b/dhpython/build/plugin_distutils.py new file mode 100644 index 0000000..342ec6f --- /dev/null +++ b/dhpython/build/plugin_distutils.py @@ -0,0 +1,121 @@ +# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +from glob import glob1 +from os import remove +from os.path import exists, isdir, join +from shutil import rmtree +from dhpython.build.base import Base, shell_command, copy_test_files + +log = logging.getLogger('dhpython') +_setup_tpl = 'setup.py|setup-3.py' + + +def create_pydistutils_cfg(func): + """distutils doesn't have sane command-line API - this decorator creates + .pydistutils.cfg file to workaround it + + hint: if you think this is plain stupid, please don't read + distutils/setuptools/distribute sources + """ + + def wrapped_func(self, context, args, *oargs, **kwargs): + fpath = join(args['home_dir'], '.pydistutils.cfg') + if not exists(fpath): + with open(fpath, 'w', encoding='utf-8') as fp: + lines = ['[clean]\n', + 'all=1\n', + '[build]\n', + 'build_lib={}\n'.format(args['build_dir']), + '[install]\n', + 'force=1\n', + 'install_layout=deb\n', + 'install_scripts=$base/bin\n', + 'install_lib={}\n'.format(args['install_dir']), + 'prefix=/usr\n'] + log.debug('pydistutils config file:\n%s', ''.join(lines)) + fp.writelines(lines) + context['ENV']['HOME'] = args['home_dir'] + return func(self, context, args, *oargs, **kwargs) + + wrapped_func.__name__ = func.__name__ + return wrapped_func + + +class BuildSystem(Base): + DESCRIPTION = 'Distutils build system' + SUPPORTED_INTERPRETERS = {'python', 'python3', 'python{version}', + 'python-dbg', 'python3-dbg', 'python{version}-dbg', + 'pypy'} + REQUIRED_FILES = [_setup_tpl] + OPTIONAL_FILES = {'setup.cfg': 1, + 'requirements.txt': 1, + 'PKG-INFO': 10, + '*.egg-info': 10} + CLEAN_FILES = Base.CLEAN_FILES | {'build'} + + def detect(self, context): + result = super(BuildSystem, self).detect(context) + if _setup_tpl in self.DETECTED_REQUIRED_FILES: + context['args']['setup_py'] = self.DETECTED_REQUIRED_FILES[_setup_tpl][0] + else: + context['args']['setup_py'] = 'setup.py' + return result + + @shell_command + @create_pydistutils_cfg + def clean(self, context, args): + super(BuildSystem, self).clean(context, args) + if exists(args['interpreter'].binary()): + return '{interpreter} {setup_py} clean {args}' + return 0 # no need to invoke anything + + @shell_command + @create_pydistutils_cfg + def configure(self, context, args): + return '{interpreter} {setup_py} config {args}' + + @shell_command + @create_pydistutils_cfg + def build(self, context, args): + return '{interpreter.binary_dv} {setup_py} build {args}' + + @shell_command + @create_pydistutils_cfg + def install(self, context, args): + # remove egg-info dirs from build_dir + for fname in glob1(args['build_dir'], '*.egg-info'): + fpath = join(args['build_dir'], fname) + rmtree(fpath) if isdir(fpath) else remove(fpath) + + return '{interpreter.binary_dv} {setup_py} install --root {destdir} {args}' + + @shell_command + @create_pydistutils_cfg + @copy_test_files() + def test(self, context, args): + if not self.cfg.custom_tests: + fpath = join(args['dir'], args['setup_py']) + with open(fpath, 'rb') as fp: + if fp.read().find(b'test_suite') > 0: + # TODO: is that enough to detect if test target is available? + return '{interpreter} {setup_py} test {args}' + return super(BuildSystem, self).test(context, args) diff --git a/dhpython/build/plugin_flit.py b/dhpython/build/plugin_flit.py new file mode 100644 index 0000000..004d657 --- /dev/null +++ b/dhpython/build/plugin_flit.py @@ -0,0 +1,170 @@ +# Copyright © 2012-2020 Piotr Ożarowski <piotr@debian.org> +# © 2020 Scott Kitterman <scott@kitterman.com> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from fnmatch import fnmatch +from pathlib import Path +import copy +import csv +import logging +import os +import os.path as osp +import shutil +import sysconfig +try: + import tomli +except ModuleNotFoundError: + # Plugin still works, only needed for autodetection + pass +try: + from flit.install import Installer +except ImportError: + Installer = object + +from dhpython.build.base import Base, shell_command + +log = logging.getLogger('dhpython') + + +class DebianInstaller(Installer): + def install_directly(self, destdir, installdir): + """Install a module/package into package directory, and create its + scripts. + """ + if installdir[:1] == os.sep: + installdir = installdir[1:] + + vars_ = copy.copy(sysconfig.get_config_vars()) + vars_['base'] = destdir + vars_['base'] + try: + dirs = sysconfig.get_paths(scheme='deb_system', vars=vars_) + except KeyError: + # Debian hasn't patched sysconfig schemes until 3.10 + # TODO: Introduce a version check once sysconfig is patched. + dirs = sysconfig.get_paths(scheme='posix_prefix', vars=vars_) + + dirs['purelib'] = dirs['platlib'] = osp.join(destdir, installdir) + os.makedirs(dirs['purelib'], exist_ok=True) + os.makedirs(dirs['scripts'], exist_ok=True) + + dst = osp.join(dirs['purelib'], osp.basename(self.module.path)) + if osp.lexists(dst): + if osp.isdir(dst) and not osp.islink(dst): + shutil.rmtree(dst) + else: + os.unlink(dst) + + src = str(self.module.path) + if self.module.is_package: + log.info("Installing package %s -> %s", src, dst) + shutil.copytree(src, dst) + self._record_installed_directory(dst) + else: + log.info("Installing file %s -> %s", src, dst) + shutil.copy2(src, dst) + self.installed_files.append(dst) + + scripts = self.ini_info.entrypoints.get('console_scripts', {}) + if scripts: + log.info("Installing scripts to %s", dirs['scripts']) + self.install_scripts(scripts, dirs['scripts']) + + log.info("Writing dist-info %s", dirs['purelib']) + self.write_dist_info(dirs['purelib']) + # Remove direct_url.json - contents are not useful or reproduceable + for path in Path(dirs['purelib']).glob("*.dist-info/direct_url.json"): + path.unlink() + # Remove build path from RECORD files + for path in Path(dirs['purelib']).glob("*.dist-info/RECORD"): + with open(path) as f: + reader = csv.reader(f) + records = list(reader) + with open(path, 'w') as f: + writer = csv.writer(f) + for path, hash_, size in records: + path = path.replace(destdir, '') + if fnmatch(path, "*.dist-info/direct_url.json"): + continue + writer.writerow([path, hash_, size]) + + +class BuildSystem(Base): + DESCRIPTION = 'Flit build system' + SUPPORTED_INTERPRETERS = {'python3', 'python{version}'} + REQUIRED_FILES = ['pyproject.toml'] + OPTIONAL_FILES = {} + CLEAN_FILES = Base.CLEAN_FILES | {'build'} + + def detect(self, context): + """Return certainty level that this plugin describes the right build + system + + This method uses cls.{REQUIRED}_FILES (pyroject.toml) as well as + checking to see if build-backend is set to flit_core.buildapi. + + Score is 85 if both are present (to allow manually setting distutils to + score higher if set). + + :return: 0 <= certainty <= 100 + :rtype: int + """ + if Installer is object: + return 0 + + result = super().detect(context) + if result > 100: + return 100 + return result + + def clean(self, context, args): + super().clean(context, args) + if osp.exists(args['interpreter'].binary()): + log.debug("removing '%s' (and everything under it)", + args['build_dir']) + osp.isdir(args['build_dir']) and shutil.rmtree(args['build_dir']) + return 0 # no need to invoke anything + + def configure(self, context, args): + # Flit does not support binary extensions + return 0 # Not needed for flit + + def build(self, context, args): + log.warning("The pybuild flit plugin is deprecated, " + "please use the pyproject plugin instead.") + my_dir = Path(args['dir']) + install_kwargs = {'user': False, 'symlink': False, 'deps': 'none'} + DebianInstaller.from_ini_path(my_dir / 'pyproject.toml', + **install_kwargs).install_directly( + args['build_dir'], '') + # These get byte compiled too, although it's not logged. + return 0 # Not needed for flit + + def install(self, context, args): + my_dir = Path(args['dir']) + install_kwargs = {'user': False, 'symlink': False, 'deps': 'none'} + DebianInstaller.from_ini_path(my_dir / 'pyproject.toml', + **install_kwargs).install_directly( + args['destdir'], + args['install_dir']) + return 0 # Not needed for flit' + + @shell_command + def test(self, context, args): + return super().test(context, args) diff --git a/dhpython/build/plugin_pyproject.py b/dhpython/build/plugin_pyproject.py new file mode 100644 index 0000000..b24aa9a --- /dev/null +++ b/dhpython/build/plugin_pyproject.py @@ -0,0 +1,201 @@ +# Copyright © 2012-2020 Piotr Ożarowski <piotr@debian.org> +# © 2020 Scott Kitterman <scott@kitterman.com> +# © 2021 Stuart Prescott <stuart@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from pathlib import Path +import logging +import os.path as osp +import shutil +import sysconfig +try: + import tomli +except ModuleNotFoundError: + # Plugin still works, only needed for autodetection + pass +try: + from installer import install + from installer.destinations import SchemeDictionaryDestination + from installer.sources import WheelFile +except ModuleNotFoundError: + SchemeDictionaryDestination = WheelFile = install = None + +from dhpython.build.base import Base, shell_command + +log = logging.getLogger('dhpython') + + +class BuildSystem(Base): + DESCRIPTION = 'Generic PEP517 build system' + SUPPORTED_INTERPRETERS = {'python3', 'python{version}'} + REQUIRED_FILES = ['pyproject.toml'] + OPTIONAL_FILES = {} + CLEAN_FILES = Base.CLEAN_FILES | {'build'} + + def detect(self, context): + """Return certainty level that this plugin describes the right build + system + + This method uses cls.{REQUIRED}_FILES (pyroject.toml) only; any + other PEP517 compliant builder (such as the flit) builder should + indicate higher specificity than this plugin. + + :return: 0 <= certainty <= 100 + :rtype: int + """ + result = super().detect(context) + # Temporarily reduce the threshold while we're in beta + result -= 20 + + try: + with open('pyproject.toml', 'rb') as f: + pyproject = tomli.load(f) + if pyproject.get('build-system', {}).get('build-backend'): + result += 10 + else: + # Not a PEP517 built package + result = 0 + except NameError: + # No toml, no autdetection + result = 0 + except FileNotFoundError: + # Not a PEP517 package + result = 0 + if result > 100: + return 100 + return result + + def clean(self, context, args): + super().clean(context, args) + if osp.exists(args['interpreter'].binary()): + log.debug("removing '%s' (and everything under it)", + args['build_dir']) + osp.isdir(args['build_dir']) and shutil.rmtree(args['build_dir']) + return 0 # no need to invoke anything + + def configure(self, context, args): + if install is None: + raise Exception("PEP517 plugin dependencies are not available. " + "Please Build-Depend on pybuild-plugin-pyproject.") + # No separate configure step + return 0 + + def build(self, context, args): + self.build_step1(context, args) + self.build_step2(context, args) + + @shell_command + def build_step1(self, context, args): + """ build a wheel using the PEP517 builder defined by upstream """ + log.info('Building wheel for %s with "build" module', + args['interpreter']) + context['ENV']['FLIT_NO_NETWORK'] = '1' + context['ENV']['HOME'] = args['home_dir'] + return ('{interpreter} -m build ' + '--skip-dependency-check --no-isolation --wheel ' + '--outdir ' + args['home_dir'] + + ' {args}' + ) + + def build_step2(self, context, args): + """ unpack the wheel into pybuild's normal """ + log.info('Unpacking wheel built for %s with "installer" module', + args['interpreter']) + extras = {} + for extra in ('scripts', 'data'): + path = Path(args["home_dir"]) / extra + if osp.exists(path): + log.warning(f'{extra.title()} directory already exists, ' + 'skipping unpack. ' + 'Is the Python package being built twice?') + return + extras[extra] = path + destination = SchemeDictionaryDestination( + { + 'platlib': args['build_dir'], + 'purelib': args['build_dir'], + 'scripts': extras['scripts'], + 'data': extras['data'], + }, + interpreter=args['interpreter'].binary_dv, + script_kind='posix', + ) + + # FIXME this next step will unpack every single wheel file it finds + # which is probably ok since each wheel is built in a separate + # directory; but perhaps it should only accept the correctly named + # wheels that match the current interpreter? + # python-packaging has relevant utilities in + # - packaging/utils.py::parse_wheel_filename + # - packaging/tags.py (although it is current-interpreter-centric) + wheels = Path(args['home_dir']).glob('*.whl') + for wheel in wheels: + if wheel.name.startswith('UNKNOWN'): + raise Exception(f'UNKNOWN wheel found: {wheel.name}. Does ' + 'pyproject.toml specify a build-backend?') + with WheelFile.open(wheel) as source: + install( + source=source, + destination=destination, + additional_metadata={}, + ) + + def install(self, context, args): + log.info('Copying package built for %s to destdir', + args['interpreter']) + try: + paths = sysconfig.get_paths(scheme='deb_system') + except KeyError: + # Debian hasn't patched sysconfig schemes until 3.10 + # TODO: Introduce a version check once sysconfig is patched. + paths = sysconfig.get_paths(scheme='posix_prefix') + + # start by copying the data and scripts + for extra in ('data', 'scripts'): + src_dir = Path(args['home_dir']) / extra + if not src_dir.exists(): + continue + target_dir = args['destdir'] + paths[extra] + log.debug('Copying %s directory contents from %s -> %s', + extra, src_dir, target_dir) + shutil.copytree( + src_dir, + target_dir, + dirs_exist_ok=True, + ) + + # then copy the modules + module_dir = args['build_dir'] + target_dir = args['destdir'] + args['install_dir'] + log.debug('Copying module contents from %s -> %s', + module_dir, target_dir) + shutil.copytree( + module_dir, + target_dir, + dirs_exist_ok=True, + ) + + @shell_command + def test(self, context, args): + scripts = Path(args["home_dir"]) / 'scripts' + if scripts.exists(): + context['ENV']['PATH'] = f"{scripts}:{context['ENV']['PATH']}" + context['ENV']['HOME'] = args['home_dir'] + return super().test(context, args) diff --git a/dhpython/debhelper.py b/dhpython/debhelper.py new file mode 100644 index 0000000..f497a60 --- /dev/null +++ b/dhpython/debhelper.py @@ -0,0 +1,327 @@ +# Copyright © 2010-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import errno +import logging +import re +from os import makedirs, chmod, environ +from os.path import basename, exists, join, dirname +from sys import argv +from dhpython import DEPENDS_SUBSTVARS, PKG_NAME_TPLS, RT_LOCATIONS, RT_TPLS + +log = logging.getLogger('dhpython') +parse_dep = re.compile('''[,\s]* + (?P<name>[^ :]+)(?::any)? + \s* + \(?(?P<version>([>=<]{2,}|=)\s*[^\)]+)?\)? + \s* + (?:\[(?P<arch>[^\]]+)\])? + ''', re.VERBOSE).match + + +def build_options(**options): + """Build an Options object from kw options""" + default_options = { + 'arch': None, + 'package': [], + 'no_package': [], + 'write_log': False, + 'remaining_packages': False, + } + built_options = default_options + built_options.update(options) + return type('Options', (object,), built_options) + + +class DebHelper: + """Reinvents the wheel / some dh functionality (Perl is ugly ;-P)""" + + def __init__(self, options, impl='cpython3'): + self.options = options + self.packages = {} + self.build_depends = {} + self.python_version = None + # Note that each DebHelper instance supports ONE interpreter type only + # it's not possible to mix cpython2, cpython3 and pypy here + self.impl = impl + self.command = { + 'cpython2': 'dh_python2', + 'cpython3': 'dh_python3', + 'pypy': 'dh_pypy', + }[impl] + skip_tpl = set() + for name, tpls in PKG_NAME_TPLS.items(): + if name != impl: + skip_tpl.update(tpls) + skip_tpl = tuple(skip_tpl) + substvar = DEPENDS_SUBSTVARS[impl] + + pkgs = options.package + skip_pkgs = options.no_package + + try: + with open('debian/control', 'r', encoding='utf-8') as fp: + paragraphs = [{}] + field = None + for lineno, line in enumerate(fp, 1): + if line.startswith('#'): + continue + if not line.strip(): + if paragraphs[-1]: + paragraphs.append({}) + field = None + continue + if line[0].isspace(): # Continuation + paragraphs[-1][field] += line.rstrip() + continue + if not ':' in line: + raise Exception( + 'Unable to parse line %i in debian/control: %s' + % (lineno, line)) + field, value = line.split(':', 1) + field = field.lower() + paragraphs[-1][field] = value.strip() + except IOError: + raise Exception('cannot find debian/control file') + + # Trailing new lines? + if not paragraphs[-1]: + paragraphs.pop() + + if len(paragraphs) < 2: + raise Exception('Unable to parse debian/control, found less than ' + '2 paragraphs') + + self.source_name = paragraphs[0]['source'] + if self.impl == 'cpython3' and 'x-python3-version' in paragraphs[0]: + self.python_version = paragraphs[0]['x-python3-version'] + if len(self.python_version.split(',')) > 2: + raise ValueError('too many arguments provided for ' + 'X-Python3-Version: min and max only.') + elif self.impl == 'cpython2': + if 'x-python-version' in paragraphs[0]: + self.python_version = paragraphs[0]['x-python-version'] + elif 'xs-python-version' in paragraphs[0]: + self.python_version = paragraphs[0]['xs-python-version'] + + build_depends = [] + for field in ('build-depends', 'build-depends-indep', + 'build-depends-arch'): + if field in paragraphs[0]: + build_depends.append(paragraphs[0][field]) + build_depends = ', '.join(build_depends) + for dep1 in build_depends.split(','): + for dep2 in dep1.split('|'): + details = parse_dep(dep2) + if details: + details = details.groupdict() + if details['arch']: + architectures = details['arch'].split() + else: + architectures = [None] + for arch in architectures: + self.build_depends.setdefault( + details['name'], {})[arch] = details['version'] + + for paragraph_no, paragraph in enumerate(paragraphs[1:], 2): + if 'package' not in paragraph: + raise Exception('Unable to parse debian/control, paragraph %i ' + 'missing Package field' % paragraph_no) + binary_package = paragraph['package'] + if skip_tpl and binary_package.startswith(skip_tpl): + log.debug('skipping package: %s', binary_package) + continue + if pkgs and binary_package not in pkgs: + continue + if skip_pkgs and binary_package in skip_pkgs: + continue + if (options.remaining_packages and + self.has_acted_on_package(binary_package)): + continue + pkg = { + 'substvars': {}, + 'autoscripts': {}, + 'rtupdates': [], + 'arch': paragraph['architecture'], + } + if (options.arch is False and pkg['arch'] != 'all' or + options.arch is True and pkg['arch'] == 'all'): + # TODO: check also if arch matches current architecture: + continue + + if not binary_package.startswith(PKG_NAME_TPLS[impl]): + # package doesn't have common prefix (python-, python3-, pypy-) + # so lets check if Depends/Recommends contains the + # appropriate substvar + if (substvar not in paragraph.get('depends', '') + and substvar not in paragraph.get('recommends', '')): + log.debug('skipping package %s (missing %s in ' + 'Depends/Recommends)', + binary_package, substvar) + continue + # Operate on binary_package + self.packages[binary_package] = pkg + + fp.close() + log.debug('source=%s, binary packages=%s', self.source_name, + list(self.packages.keys())) + + def has_acted_on_package(self, package): + try: + with open('debian/{}.debhelper.log'.format(package), + encoding='utf-8') as f: + for line in f: + if line.strip() == self.command: + return True + except IOError as e: + if e.errno != errno.ENOENT: + raise + return False + + def addsubstvar(self, package, name, value): + """debhelper's addsubstvar""" + self.packages[package]['substvars'].setdefault(name, []).append(value) + + def autoscript(self, package, when, template, args): + """debhelper's autoscript""" + self.packages[package]['autoscripts'].setdefault(when, {})\ + .setdefault(template, []).append(args) + + def add_rtupdate(self, package, value): + self.packages[package]['rtupdates'].append(value) + + def save_autoscripts(self): + for package, settings in self.packages.items(): + autoscripts = settings.get('autoscripts') + if not autoscripts: + continue + + for when, templates in autoscripts.items(): + fn = "debian/%s.%s.debhelper" % (package, when) + if exists(fn): + with open(fn, 'r', encoding='utf-8') as datafile: + data = datafile.read() + else: + data = '' + + new_data = '' + for tpl_name, args in templates.items(): + for i in args: + # try local one first (useful while testing dh_python3) + fpath = join(dirname(__file__), '..', + "autoscripts/%s" % tpl_name) + if not exists(fpath): + fpath = "/usr/share/debhelper/autoscripts/%s" % tpl_name + with open(fpath, 'r', encoding='utf-8') as tplfile: + tpl = tplfile.read() + if self.options.compile_all and args: + # TODO: should args be checked to contain dir name? + tpl = tpl.replace('-p #PACKAGE#', '') + elif settings['arch'] == 'all': + tpl = tpl.replace('#PACKAGE#', package) + else: + arch = environ['DEB_HOST_ARCH'] + tpl = tpl.replace('#PACKAGE#', '%s:%s' % (package, arch)) + tpl = tpl.replace('#ARGS#', i) + if tpl not in data and tpl not in new_data: + new_data += "\n%s" % tpl + if new_data: + data += '\n# Automatically added by {}'.format(basename(argv[0])) +\ + '{}\n# End automatically added section\n'.format(new_data) + fp = open(fn, 'w', encoding='utf-8') + fp.write(data) + fp.close() + + def save_substvars(self): + for package, settings in self.packages.items(): + substvars = settings.get('substvars') + if not substvars: + continue + fn = "debian/%s.substvars" % package + if exists(fn): + with open(fn, 'r', encoding='utf-8') as datafile: + data = datafile.read() + else: + data = '' + for name, values in substvars.items(): + p = data.find("%s=" % name) + if p > -1: # parse the line and remove it from data + e = data[p:].find('\n') + line = data[p + len("%s=" % name): + p + e if e > -1 else None] + items = [i.strip() for i in line.split(',') if i] + if e > -1 and data[p + e:].strip(): + data = "%s\n%s" % (data[:p], data[p + e:]) + else: + data = data[:p] + else: + items = [] + for j in values: + if j not in items: + items.append(j) + if items: + if data: + data += '\n' + data += "%s=%s\n" % (name, ', '.join(items)) + data = data.replace('\n\n', '\n') + if data: + fp = open(fn, 'w', encoding='utf-8') + fp.write(data) + fp.close() + + def save_rtupdate(self): + for package, settings in self.packages.items(): + pkg_arg = '' if self.options.compile_all else "-p %s" % package + values = settings.get('rtupdates') + if not values: + continue + d = 'debian/{}/{}'.format(package, RT_LOCATIONS[self.impl]) + if not exists(d): + makedirs(d) + fn = "%s/%s.rtupdate" % (d, package) + if exists(fn): + data = open(fn, 'r', encoding='utf-8').read() + else: + data = "#! /bin/sh\nset -e" + for dname, args in values: + cmd = RT_TPLS[self.impl].format(pkg_arg=pkg_arg, + dname=dname, + args=args) + if cmd not in data: + data += "\n%s" % cmd + if data: + fp = open(fn, 'w', encoding='utf-8') + fp.write(data) + fp.close() + chmod(fn, 0o755) + + def save_log(self): + if not self.options.write_log: + return + for package, settings in self.packages.items(): + with open('debian/{}.debhelper.log'.format(package), + 'a', encoding='utf-8') as f: + f.write(self.command + '\n') + + def save(self): + self.save_substvars() + self.save_autoscripts() + self.save_rtupdate() + self.save_log() diff --git a/dhpython/depends.py b/dhpython/depends.py new file mode 100644 index 0000000..e17951c --- /dev/null +++ b/dhpython/depends.py @@ -0,0 +1,281 @@ +# Copyright © 2010-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +from functools import partial +from os.path import exists, join +from dhpython import PKG_PREFIX_MAP, MINPYCDEP +from dhpython.pydist import parse_pydep, parse_requires_dist, guess_dependency +from dhpython.version import default, supported, VersionRange + +log = logging.getLogger('dhpython') + + +class Dependencies: + """Store relations (dependencies, etc.) between packages.""" + + def __init__(self, package, impl='cpython3', bdep=None): + self.impl = impl + self.package = package + bdep = self.bdep = bdep or {} + self.is_debug_package = dbgpkg = package.endswith('-dbg') + + # TODO: move it to PyPy and CPython{2,3} classes + self.ipkg_vtpl = 'python%s-dbg' if dbgpkg else 'python%s' + if impl == 'cpython3': + self.ipkg_tpl = 'python3-dbg' if dbgpkg else 'python3' + elif impl == 'cpython2': + self.ipkg_tpl = 'python2-dbg' if dbgpkg else 'python2' + elif impl == 'pypy': + self.ipkg_tpl = 'pypy-dbg' if dbgpkg else 'pypy' + self.ipkg_vtpl = 'pypy%s-dbg' if dbgpkg else 'pypy%s' + if impl == 'pypy': + self.ipkg_tpl_ma = self.ipkg_tpl + self.ipkg_vtpl_ma = self.ipkg_vtpl + else: + self.ipkg_tpl_ma = self.ipkg_tpl + ':any' + self.ipkg_vtpl_ma = self.ipkg_vtpl + ':any' + + self.python_dev_in_bd = 'python-dev' in bdep or\ + 'python-all-dev' in bdep or\ + 'python2-dev' in bdep or\ + 'python2-all-dev' in bdep or\ + 'python2.7-dev' in bdep or\ + 'python3-dev' in bdep or\ + 'python3-all-dev' in bdep + + self.depends = set() + self.recommends = [] + self.suggests = [] + self.enhances = [] + self.breaks = [] + self.rtscripts = [] + + def export_to(self, dh): + """Fill in debhelper's substvars.""" + prefix = PKG_PREFIX_MAP.get(self.impl, 'misc') + for i in sorted(self.depends): + dh.addsubstvar(self.package, '{}:Depends'.format(prefix), i) + for i in sorted(self.recommends): + dh.addsubstvar(self.package, '{}:Recommends'.format(prefix), i) + for i in sorted(self.suggests): + dh.addsubstvar(self.package, '{}:Suggests'.format(prefix), i) + for i in sorted(self.enhances): + dh.addsubstvar(self.package, '{}:Enhances'.format(prefix), i) + for i in sorted(self.breaks): + dh.addsubstvar(self.package, '{}:Breaks'.format(prefix), i) + for i in sorted(self.rtscripts): + dh.add_rtupdate(self.package, i) + + def __str__(self): + return "D=%s; R=%s; S=%s; E=%s, B=%s; RT=%s" %\ + (self.depends, self.recommends, self.suggests, + self.enhances, self.breaks, self.rtscripts) + + def depend(self, value): + if value and value not in self.depends: + self.depends.add(value) + + def recommend(self, value): + if value and value not in self.recommends: + self.recommends.append(value) + + def suggest(self, value): + if value and value not in self.suggests: + self.suggests.append(value) + + def enhance(self, value): + if value and value not in self.enhances: + self.enhances.append(value) + + def break_(self, value): + if value and value not in self.breaks: + self.breaks.append(value) + + def rtscript(self, value): + if value not in self.rtscripts: + self.rtscripts.append(value) + + def parse(self, stats, options): + log.debug('generating dependencies for package %s', self.package) + tpl = self.ipkg_tpl + vtpl = self.ipkg_vtpl + tpl_ma = self.ipkg_tpl_ma + vtpl_ma = self.ipkg_vtpl_ma + vrange = options.vrange + + if vrange and any((stats['compile'], stats['public_vers'], + stats['ext_vers'], stats['ext_no_version'], + stats['shebangs'])): + if any((stats['compile'], stats['public_vers'], stats['shebangs'])): + tpl_tmp = tpl_ma + else: + tpl_tmp = tpl + minv = vrange.minver + # note it's an open interval (i.e. do not add 1 here!): + maxv = vrange.maxver + if minv == maxv: + self.depend(vtpl % minv) + minv = maxv = None + if minv: + self.depend("%s (>= %s~)" % (tpl_tmp, minv)) + if maxv: + self.depend("%s (<< %s)" % (tpl_tmp, maxv)) + + if self.impl == 'cpython2' and stats['public_vers']: + # additional Depends to block python package transitions + sorted_vers = sorted(stats['public_vers']) + minv = sorted_vers[0] + maxv = sorted_vers[-1] + if minv <= default(self.impl): + self.depend("%s (>= %s~)" % (tpl_ma, minv)) + if maxv >= default(self.impl): + self.depend("%s (<< %s)" % (tpl_ma, maxv + 1)) + + if self.impl == 'pypy' and stats.get('ext_soabi'): + # TODO: make sure alternative is used only for the same extension names + # ie. for foo.ABI1.so, foo.ABI2.so, bar.ABI3,so, bar.ABI4.so generate: + # pypy-abi-ABI1 | pypy-abi-ABI2, pypy-abi-ABI3 | pypy-abi-ABI4 + self.depend('|'.join(soabi.replace('-', '-abi-') + for soabi in sorted(stats['ext_soabi']))) + + if stats['ext_vers']: + # TODO: what about extensions with stable ABI? + sorted_vers = sorted(stats['ext_vers']) + minv = sorted_vers[0] + maxv = sorted_vers[-1] + #self.depend('|'.join(vtpl % i for i in stats['ext_vers'])) + if minv <= default(self.impl): + self.depend("%s (>= %s~)" % (tpl, minv)) + if maxv >= default(self.impl): + self.depend("%s (<< %s)" % (tpl, maxv + 1)) + + # make sure py{,3}compile binary is available + if stats['compile'] and self.impl in MINPYCDEP: + self.depend(MINPYCDEP[self.impl]) + + for ipreter in stats['shebangs']: + self.depend("%s%s" % (ipreter, '' if self.impl == 'pypy' else ':any')) + + supported_versions = supported(self.impl) + default_version = default(self.impl) + for private_dir, details in stats['private_dirs'].items(): + versions = list(i.version for i in details.get('shebangs', []) if i.version and i.version.minor) + + for v in versions: + if v in supported_versions: + self.depend(vtpl_ma % v) + else: + log.info('dependency on %s (from shebang) ignored' + ' - it\'s not supported anymore', vtpl % v) + # /usr/bin/python{,3} shebang → add python{,3} to Depends + if any(True for i in details.get('shebangs', []) if i.version is None or i.version.minor is None): + self.depend(tpl_ma) + + extensions = False + if self.python_dev_in_bd: + extensions = sorted(details.get('ext_vers', set())) + #self.depend('|'.join(vtpl % i for i in extensions)) + if extensions: + self.depend("%s (>= %s~)" % (tpl, extensions[0])) + self.depend("%s (<< %s)" % (tpl, extensions[-1] + 1)) + elif details.get('ext_no_version'): + # assume unrecognized extension was built for default interpreter version + self.depend("%s (>= %s~)" % (tpl, default_version)) + self.depend("%s (<< %s)" % (tpl, default_version + 1)) + + if details.get('compile'): + if self.impl in MINPYCDEP: + self.depend(MINPYCDEP[self.impl]) + args = '' + if extensions: + args += "-V %s" % VersionRange(minver=extensions[0], maxver=extensions[-1]) + elif len(versions) == 1: # only one version from shebang + #if versions[0] in supported_versions: + args += "-V %s" % versions[0] + # ... otherwise compile with default version + elif details.get('ext_no_version'): + # assume unrecognized extension was built for default interpreter version + args += "-V %s" % default_version + elif vrange: + args += "-V %s" % vrange + if vrange.minver == vrange.maxver: + self.depend(vtpl % vrange.minver) + else: + if vrange.minver: # minimum version specified + self.depend("%s (>= %s~)" % (tpl_ma, vrange.minver)) + if vrange.maxver: # maximum version specified + self.depend("%s (<< %s)" % (tpl_ma, vrange.maxver + 1)) + + for regex in options.regexpr or []: + args += " -X '%s'" % regex.pattern.replace("'", r"'\''") + self.rtscript((private_dir, args)) + + section_options = { + 'depends_sec': options.depends_section, + 'recommends_sec': options.recommends_section, + 'suggests_sec': options.suggests_section, + } + guess_deps = partial(guess_dependency, impl=self.impl, bdep=self.bdep, + accept_upstream_versions=options.accept_upstream_versions) + if options.guess_deps: + for fn in stats['requires.txt']: + # TODO: should options.recommends and options.suggests be + # removed from requires.txt? + deps = parse_pydep(self.impl, fn, bdep=self.bdep, **section_options) + [self.depend(i) for i in deps['depends']] + [self.recommend(i) for i in deps['recommends']] + [self.suggest(i) for i in deps['suggests']] + for fpath in stats['egg-info']: + with open(fpath, 'r', encoding='utf-8') as fp: + for line in fp: + if line.startswith('Requires: '): + req = line[10:].strip() + self.depend(guess_deps(req=req)) + for fpath in stats['dist-info']: + deps = parse_requires_dist(self.impl, fpath, bdep=self.bdep, + **section_options) + [self.depend(i) for i in deps['depends']] + [self.recommend(i) for i in deps['recommends']] + [self.suggest(i) for i in deps['suggests']] + + # add dependencies from --depends + for item in options.depends or []: + self.depend(guess_deps(req=item)) + # add dependencies from --recommends + for item in options.recommends or []: + self.recommend(guess_deps(req=item)) + # add dependencies from --suggests + for item in options.suggests or []: + self.suggest(guess_deps(req=item)) + # add dependencies from --requires + for fn in options.requires or []: + fpath = join('debian', self.package, fn) + if not exists(fpath): + fpath = fn + if not exists(fpath): + log.warn('cannot find requirements file: %s', fn) + continue + deps = parse_pydep(self.impl, fpath, bdep=self.bdep, **section_options) + [self.depend(i) for i in deps['depends']] + [self.recommend(i) for i in deps['recommends']] + [self.suggest(i) for i in deps['suggests']] + + log.debug(self) diff --git a/dhpython/exceptions.py b/dhpython/exceptions.py new file mode 100644 index 0000000..085bb89 --- /dev/null +++ b/dhpython/exceptions.py @@ -0,0 +1,23 @@ +# Copyright © 2022 Stefano Rivera <stefanor@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + +class RequiredCommandMissingException(Exception): + pass diff --git a/dhpython/fs.py b/dhpython/fs.py new file mode 100644 index 0000000..445422e --- /dev/null +++ b/dhpython/fs.py @@ -0,0 +1,587 @@ +# Copyright © 2013-2019 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import difflib +import hashlib +import logging +import os +import re +import sys +from filecmp import cmp as cmpfile +from glob import glob +from os.path import (lexists, exists, getsize, isdir, islink, join, realpath, + relpath, split, splitext) +from shutil import rmtree +from stat import ST_MODE, S_IXUSR, S_IXGRP, S_IXOTH +from dhpython import MULTIARCH_DIR_TPL +from dhpython.tools import fix_shebang, clean_egg_name +from dhpython.interpreter import Interpreter + +log = logging.getLogger('dhpython') + + +def fix_locations(package, interpreter, versions, options): + """Move files to the right location.""" + # make a copy since we change version later + interpreter = Interpreter(interpreter) + + for version in versions: + interpreter.version = version + + dstdir = interpreter.sitedir(package) + for srcdir in interpreter.old_sitedirs(package): + if isdir(srcdir): + # TODO: what about relative symlinks? + log.debug('moving files from %s to %s', srcdir, dstdir) + share_files(srcdir, dstdir, interpreter, options) + try: + os.removedirs(srcdir) + except OSError: + pass + + # do the same with debug locations + dstdir = interpreter.sitedir(package, gdb=True) + for srcdir in interpreter.old_sitedirs(package, gdb=True): + if isdir(srcdir): + log.debug('moving files from %s to %s', srcdir, dstdir) + share_files(srcdir, dstdir, interpreter, options) + try: + os.removedirs(srcdir) + except OSError: + pass + + # move files from /usr/include/pythonX.Y/ to …/pythonX.Ym/ + if interpreter.symlinked_include_dir: + srcdir = "debian/%s%s" % (package, interpreter.symlinked_include_dir) + if srcdir and isdir(srcdir): + dstdir = "debian/%s%s" % (package, interpreter.include_dir) + log.debug('moving files from %s to %s', srcdir, dstdir) + share_files(srcdir, dstdir, interpreter, options) + try: + os.removedirs(srcdir) + except OSError: + pass + + +def share_files(srcdir, dstdir, interpreter, options): + """Try to move as many files from srcdir to dstdir as possible.""" + cleanup_actions = [] + for i in os.listdir(srcdir): + fpath1 = join(srcdir, i) + if not lexists(fpath1): # removed in rename_ext + continue + if i.endswith('.pyc'): # f.e. when tests were invoked on installed files + os.remove(fpath1) + continue + if not options.no_ext_rename and splitext(i)[-1] == '.so': + # try to rename extension here as well (in :meth:`scan` info about + # Python version is gone) + version = interpreter.parse_public_dir(srcdir) + if version and version is not True: + fpath1 = Scan.rename_ext(fpath1, interpreter, version) + i = split(fpath1)[-1] + if srcdir.endswith(".dist-info"): + if i in ('COPYING', 'LICENSE') or i.startswith( + ('COPYING.', 'LICENSE.')): + os.remove(fpath1) + cleanup_actions.append((remove_from_RECORD, ([i],))) + continue + elif isdir(fpath1) and i in ('licenses', 'license_files'): + cleanup_actions.append(( + remove_from_RECORD, + ([ + relpath(license, srcdir) + for license in glob(join(srcdir, i, '**')) + ],) + )) + rmtree(fpath1) + continue + fpath2 = join(dstdir, i) + if not isdir(fpath1) and not exists(fpath2): + # do not rename directories here - all .so files have to be renamed first + os.renames(fpath1, fpath2) + continue + if islink(fpath1): + # move symlinks without changing them if they point to the same place + if not exists(fpath2): + os.renames(fpath1, fpath2) + elif realpath(fpath1) == realpath(fpath2): + os.remove(fpath1) + elif isdir(fpath1): + share_files(fpath1, fpath2, interpreter, options) + elif cmpfile(fpath1, fpath2, shallow=False): + os.remove(fpath1) + elif i.endswith(('.abi3.so', '.abi4.so')) and interpreter.parse_public_dir(srcdir): + log.warning('%s differs from previous one, removing anyway (%s)', i, srcdir) + os.remove(fpath1) + elif srcdir.endswith(".dist-info"): + # dist-info file that differs... try merging + if i == "WHEEL": + if merge_WHEEL(fpath1, fpath2): + cleanup_actions.append((fix_merged_RECORD, ())) + os.remove(fpath1) + elif i == "RECORD": + merge_RECORD(fpath1, fpath2) + os.remove(fpath1) + else: + log.warn("No merge driver for dist-info file %s", i) + else: + # The files differed so we cannot collapse them. + log.warn('Paths differ: %s and %s', fpath1, fpath2) + if options.verbose and not i.endswith(('.so', '.a')): + with open(fpath1) as fp1: + fromlines = fp1.readlines() + with open(fpath2) as fp2: + tolines = fp2.readlines() + diff = difflib.unified_diff(fromlines, tolines, fpath1, fpath2) + sys.stderr.writelines(diff) + + for action, args in cleanup_actions: + action(dstdir, *args) + try: + os.removedirs(srcdir) + except OSError: + pass + + +## Functions to merge parts of the .dist-info metadata directory together + +def missing_lines(src, dst): + """Find all the lines in the text file src that are not in dst""" + with open(dst) as fh: + current = {k: None for k in fh.readlines()} + + missing = [] + with open(src) as fh: + for line in fh.readlines(): + if line not in current: + missing.append(line) + + return missing + + +def merge_WHEEL(src, dst): + """Merge the source .dist-info/WHEEL file into the destination + + Note that after editing the WHEEL file, the sha256 included in + the .dist-info/RECORD file will be incorrect and will need fixing + using the fix_merged_RECORD() function. + """ + log.debug("Merging WHEEL file %s into %s", src, dst) + missing = missing_lines(src, dst) + with open(dst, "at") as fh: + for line in missing: + if line.startswith("Tag: "): + fh.write(line) + else: + log.warn("WHEEL merge discarded line %s", line) + + return len(missing) + + +def merge_RECORD(src, dst): + """Merge the source .dist-info/RECORD file into the destination""" + log.debug("Merging RECORD file %s into %s", src, dst) + missing = missing_lines(src, dst) + + with open(dst, "at") as fh: + for line in missing: + fh.write(line) + + return len(missing) + + +def fix_merged_RECORD(distdir): + """Update the checksum for .dist-info/WHEEL in .dist-info/RECORD + + After merging the .dist-info/WHEEL file, the sha256 recorded for it will be + wrong in .dist-info/RECORD, so edit that file to ensure that it is fixed. + The output is sorted for reproducibility. + """ + log.debug("Fixing RECORD file in %s", distdir) + record_path = join(distdir, "RECORD") + wheel_path = join(distdir, "WHEEL") + wheel_dir = split(split(record_path)[0])[1] + wheel_relpath = join(wheel_dir, "WHEEL") + + with open(wheel_path, "rb") as fh: + wheel_sha256 = hashlib.sha256(fh.read()).hexdigest(); + wheel_size = getsize(wheel_path) + + contents = [ + "{name},sha256={sha256sum},{size}\n".format( + name=wheel_relpath, + sha256sum=wheel_sha256, + size=wheel_size, + )] + with open(record_path) as fh: + for line in fh.readlines(): + if not line.startswith(wheel_relpath): + contents.append(line) + # now write out the updated record + with open(record_path, "wt") as fh: + fh.writelines(sorted(contents)) + + +def remove_from_RECORD(distdir, files): + """Remove all specified dist-info files from RECORD""" + log.debug("Removing %r from RECORD in %s", files, distdir) + record = join(distdir, "RECORD") + parent_dir = split(distdir)[1] + names = [join(parent_dir, name) for name in files] + lines = [] + with open(record) as fh: + lines = fh.readlines() + + filtered = [line for line in lines if not line.split(',', 1)[0] in names] + + if lines == filtered: + log.warn("Unable to remove %r from RECORD in %s, not found", + files, distdir) + + with open(record, 'wt') as fh: + fh.writelines(sorted(filtered)) + + +class Scan: + UNWANTED_DIRS = re.compile(r'.*/__pycache__(/.*)?$') + UNWANTED_FILES = re.compile(r'.*\.py[co]$') + + def __init__(self, interpreter, package, dpath=None, options=None): + self.interpreter = interpreter + self.impl = interpreter.impl + + self.package = package + + if not dpath: + self.proot = "debian/%s" % self.package + else: + dpath = dpath.strip('/') + self.proot = join('debian', self.package, dpath) + self.dpath = dpath + del dpath + + self.options = options + self.result = {'requires.txt': set(), + 'egg-info': set(), + 'dist-info': set(), + 'nsp.txt': set(), + 'shebangs': set(), + 'public_vers': set(), + 'private_dirs': {}, + 'compile': False, + 'ext_vers': set(), + 'ext_no_version': set()} + + for root, dirs, file_names in os.walk(self.proot): + if interpreter.should_ignore(root): + del dirs[:] + continue + + self.current_private_dir = self.current_pub_version = None + version = interpreter.parse_public_dir(root) + if version: + self.current_dir_is_public = True + if version is True: + version = None + else: + self.current_pub_version = version + else: + self.current_dir_is_public = False + + if self.current_dir_is_public: + if root.endswith('-packages'): + if version is not None: + self.result['public_vers'].add(version) + for name in dirs: + if name in ('test', 'tests') or name.startswith('.'): + log.debug('removing dist-packages/%s', name) + rmtree(join(root, name)) + dirs.remove(name) + else: + self.current_private_dir = self.check_private_dir(root) + if not self.current_private_dir: + # i.e. not a public dir and not a private dir + if self.is_bin_dir(root): + self.handle_bin_dir(root, file_names) + else: # not a public, private or bin directory + # continue with a subdirectory + continue + + for name in dirs: + dpath = join(root, name) + if self.is_unwanted_dir(dpath): + rmtree(dpath) + dirs.remove(name) + continue + + if self.is_dist_dir(root): + self.handle_dist_dir(root, file_names) + continue + + if self.is_egg_dir(root): + self.handle_egg_dir(root, file_names) + continue + + # check files + for fn in sorted(file_names): + # sorted() to make sure .so files are handled before .so.foo + fpath = join(root, fn) + + if self.is_unwanted_file(fpath): + log.debug('removing unwanted: %s', fpath) + os.remove(fpath) + continue + + if self.is_egg_file(fpath): + self.handle_egg_file(fpath) + continue + + if not exists(fpath): + # possibly removed while handling .so symlinks + if islink(fpath) and '.so.' in split(fpath)[-1]: + # dangling symlink to (now removed/renamed) .so file + # which wasn't removed yet (see test203's quux.so.0) + log.info('removing dangling symlink: %s', fpath) + os.remove(fpath) + continue + + fext = splitext(fn)[-1][1:] + if fext == 'so': + if not self.options.no_ext_rename: + fpath = self.rename_ext(fpath, interpreter, version) + ver = self.handle_ext(fpath) + ver = ver or version + if ver: + self.current_result.setdefault('ext_vers', set()).add(ver) + else: + self.current_result.setdefault('ext_no_version', set()).add(fpath) + + if self.current_private_dir: + if exists(fpath) and fext != 'so': + mode = os.stat(fpath)[ST_MODE] + if mode & S_IXUSR or mode & S_IXGRP or mode & S_IXOTH: + if (options.no_shebang_rewrite or + fix_shebang(fpath, self.options.shebang)) and \ + not self.options.ignore_shebangs: + try: + res = Interpreter.from_file(fpath) + except Exception as e: + log.debug('cannot parse shebang %s: %s', fpath, e) + else: + self.current_result.setdefault('shebangs', set()).add(res) + + if fext == 'py' and self.handle_public_module(fpath) is not False: + self.current_result['compile'] = True + + if not dirs and not self.current_private_dir: + try: + os.removedirs(root) + except OSError: + pass + + log.debug("package %s details = %s", package, self.result) + + @property + def current_result(self): + if self.current_private_dir: + return self.result['private_dirs'].setdefault(self.current_private_dir, {}) + return self.result + + def is_unwanted_dir(self, dpath): + return self.__class__.UNWANTED_DIRS.match(dpath) + + def is_unwanted_file(self, fpath): + if self.__class__.UNWANTED_FILES.match(fpath): + return True + if self.current_dir_is_public and self.is_dbg_package\ + and self.options.clean_dbg_pkg\ + and splitext(fpath)[-1][1:] not in ('so', 'h'): + return True + + @property + def private_dirs_to_check(self): + if self.dpath: + # scan private directory *only* + return [self.dpath] + + if self.dpath is False: + result = [] + else: + result = [i % self.package for i in ( + 'usr/lib/%s', + 'usr/lib/games/%s', + 'usr/share/%s', + 'usr/share/games/%s')] + return result + + @property + def is_dbg_package(self): + #return self.interpreter.debug + return self.package.endswith('-dbg') + + def check_private_dir(self, dpath): + """Return private dir's root if it's a private dir.""" + for i in self.private_dirs_to_check: + if dpath.startswith(join('debian', self.package, i)): + return '/' + i + + @staticmethod + def rename_ext(fpath, interpreter, current_pub_version=None): + """Add multiarch triplet, etc. Return new name. + + This method is invoked for all .so files in public or private directories. + """ + # current_pub_version - version parsed from dist-packages (True if unversioned) + # i.e. if it's not None - it's a public dist-packages directory + + path, fname = fpath.rsplit('/', 1) + if current_pub_version is not None and islink(fpath): + # replace symlinks with extensions in dist-packages directory + dstfpath = fpath + links = set() + while islink(dstfpath): + links.add(dstfpath) + dstfpath = join(path, os.readlink(dstfpath)) + if exists(dstfpath) and '.so.' in split(dstfpath)[-1]: + # rename .so.$FOO symlinks, remove other ones + for lpath in links: + log.info('removing symlink: %s', lpath) + os.remove(lpath) + log.info('renaming %s to %s', dstfpath, fname) + os.rename(dstfpath, fpath) + + if MULTIARCH_DIR_TPL.match(fpath): + # ignore /lib/i386-linux-gnu/, /usr/lib/x86_64-kfreebsd-gnu/, etc. + return fpath + + new_fn = interpreter.check_extname(fname, current_pub_version) + if new_fn: + # TODO: what about symlinks pointing to this file + new_fpath = join(path, new_fn) + if exists(new_fpath): + log.warn('destination file exist, ' + 'cannot rename %s to %s', fname, new_fn) + else: + log.info('renaming %s to %s', fname, new_fn) + os.rename(fpath, new_fpath) + return new_fpath + return fpath + + def handle_ext(self, fpath): + """Handle .so file, return its version if detected.""" + + def handle_public_module(self, fpath): + pass + + def is_bin_dir(self, dpath): + """Check if dir is one from PATH ones.""" + # dname = debian/packagename/usr/games + spath = dpath.strip('/').split('/', 4) + if len(spath) > 4: + return False # assume bin directories don't have subdirectories + if dpath.endswith(('/sbin', '/bin', '/usr/games')): + # /(s)bin or /usr/(s)bin or /usr/games + return True + + def handle_bin_dir(self, dpath, file_names): + if self.options.no_shebang_rewrite or self.options.ignore_shebangs: + return + for fn in file_names: + fpath = join(dpath, fn) + if fix_shebang(fpath, self.options.shebang): + try: + res = Interpreter.from_file(fpath) + except Exception as e: + log.debug('cannot parse shebang %s: %s', fpath, e) + else: + self.result['shebangs'].add(res) + + def is_egg_dir(self, dname): + """Check if given directory contains egg-info.""" + return dname.endswith('.egg-info') + + def handle_egg_dir(self, dpath, file_names): + path, dname = dpath.rsplit('/', 1) + if self.is_dbg_package and self.options.clean_dbg_pkg: + rmtree(dpath) + return + + clean_name = clean_egg_name(dname) + if clean_name != dname: + if exists(join(path, clean_name)): + log.info('removing %s (%s is already available)', dname, clean_name) + rmtree(dpath) + return + else: + log.info('renaming %s to %s', dname, clean_name) + os.rename(dpath, join(path, clean_name)) + dname = clean_name + dpath = join(path, dname) + if file_names: + if 'requires.txt' in file_names: + self.result['requires.txt'].add(join(dpath, 'requires.txt')) + if 'namespace_packages.txt' in file_names: + self.result['nsp.txt'].add(join(dpath, 'namespace_packages.txt')) + if 'SOURCES.txt' in file_names: + os.remove(join(dpath, 'SOURCES.txt')) + file_names.remove('SOURCES.txt') + + def is_egg_file(self, fpath): + """Check if given file contains egg-info.""" + return fpath.endswith('.egg-info') + + def handle_egg_file(self, fpath): + root, name = fpath.rsplit('/', 1) + clean_name = clean_egg_name(name) + if clean_name != name: + if exists(join(root, clean_name)): + log.info('removing %s (%s is already available)', + name, clean_name) + os.remove(fpath) + else: + log.info('renaming %s to %s', name, clean_name) + os.rename(fpath, join(root, clean_name)) + self.result['egg-info'].add(join(root, clean_name)) + + def is_dist_dir(self, dname): + """Check if given directory contains dist-info.""" + return dname.endswith('.dist-info') + + def handle_dist_dir(self, dpath, file_names): + path, dname = dpath.rsplit('/', 1) + if self.is_dbg_package and self.options.clean_dbg_pkg: + rmtree(dpath) + return + + if file_names: + if 'METADATA' in file_names: + self.result['dist-info'].add(join(dpath, 'METADATA')) + + def cleanup(self): + if self.is_dbg_package and self.options.clean_dbg_pkg: + # remove empty directories in -dbg packages + proot = self.proot + '/usr/lib' + for root, dirs, file_names in os.walk(proot, topdown=False): + if '-packages/' in root and not file_names: + try: + os.removedirs(root) + except Exception: + pass diff --git a/dhpython/interpreter.py b/dhpython/interpreter.py new file mode 100644 index 0000000..021e847 --- /dev/null +++ b/dhpython/interpreter.py @@ -0,0 +1,576 @@ +# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +import os +import re +from os.path import exists, join, split +from dhpython import INTERPRETER_DIR_TPLS, PUBLIC_DIR_RE, OLD_SITE_DIRS + +SHEBANG_RE = re.compile(r''' + (?:\#!\s*){0,1} # shebang prefix + (?P<path> + .*?/bin/.*?)? + (?P<name> + python|pypy) + (?P<version> + \d[\.\d]*)? + (?P<debug> + -dbg)? + (?P<options>.*) + ''', re.VERBOSE) +EXTFILE_RE = re.compile(r''' + (?P<name>.*?) + (?:\. + (?P<stableabi>abi\d+) + |(?:\. + (?P<soabi> + (?P<impl>cpython|pypy) + - + (?P<ver>\d{2,}) + (?P<flags>[a-z]*) + )? + (?: + (?:(?<!\.)-)? # minus sign only if soabi is defined + (?P<multiarch>[^/]*?) + )? + ))? + (?P<debug>_d)? + \.so$''', re.VERBOSE) +log = logging.getLogger('dhpython') + + +class Interpreter: + """ + :attr path: /usr/bin/ in most cases + :attr name: pypy or python (even for python3 and python-dbg) or empty string + :attr version: interpreter's version + :attr debug: -dbg version of the interpreter + :attr impl: implementation (cpytho2, cpython3 or pypy) + :attr options: options parsed from shebang + :type path: str + :type name: str + :type version: Version or None + :type debug: bool + :type impl: str + :type options: tuple + """ + path = '/usr/bin/' + name = 'python' + version = None + debug = False + impl = '' + options = () + _cache = {} + + def __init__(self, value=None, path=None, name=None, version=None, + debug=None, impl=None, options=None): + params = locals() + del params['self'] + del params['value'] + + if isinstance(value, Interpreter): + for key in params.keys(): + if params[key] is None: + params[key] = getattr(value, key) + elif value: + if value.replace('.', '').isdigit() and not version: + # version string + params['version'] = Version(value) + else: + # shebang or other string + for key, val in self.parse(value).items(): + # prefer values passed to constructor over shebang ones: + if params[key] is None: + params[key] = val + + for key, val in params.items(): + if val is not None: + setattr(self, key, val) + elif key == 'version': + setattr(self, key, val) + + def __setattr__(self, name, value): + if name == 'name': + if value not in ('python', 'pypy', ''): + raise ValueError("interpreter not supported: %s" % value) + if value == 'python': + if self.version: + if self.version.major == 3: + self.__dict__['impl'] = 'cpython3' + else: + self.__dict__['impl'] = 'cpython2' + elif value == 'pypy': + self.__dict__['impl'] = 'pypy' + elif name == 'version' and value is not None: + value = Version(value) + if not self.impl and self.name == 'python': + if value.major == 3: + self.impl = 'cpython3' + else: + self.impl = 'cpython2' + if name in ('path', 'name', 'impl', 'options') and value is None: + pass + elif name == 'debug': + self.__dict__[name] = bool(value) + else: + self.__dict__[name] = value + + def __repr__(self): + result = self.path + if not result.endswith('/'): + result += '/' + result += self._vstr(self.version) + if self.options: + result += ' ' + ' '.join(self.options) + return result + + def __str__(self): + return self._vstr(self.version) + + def _vstr(self, version=None, consider_default_ver=False): + if self.impl == 'pypy': + # TODO: will Debian support more than one PyPy version? + return self.name + version = version or self.version or '' + if consider_default_ver and (not version or version == self.default_version): + version = '3' if self.impl == 'cpython3' else '2' + if self.debug: + return 'python{}-dbg'.format(version) + return self.name + str(version) + + def binary(self, version=None): + return '{}{}'.format(self.path, self._vstr(version)) + + @property + def binary_dv(self): + """Like binary(), but returns path to default intepreter symlink + if version matches default one for given implementation. + """ + return '{}{}'.format(self.path, self._vstr(consider_default_ver=True)) + + @property + def default_version(self): + if self.impl: + return default(self.impl) + + @staticmethod + def parse(shebang): + """Return dict with parsed shebang + + >>> sorted(Interpreter.parse('/usr/bin/python3.2-dbg').items()) + [('debug', '-dbg'), ('name', 'python'), ('options', ()), ('path', '/usr/bin/'), ('version', '3.2')] + >>> sorted(Interpreter.parse('#! /usr/bin/python3.2').items()) + [('debug', None), ('name', 'python'), ('options', ()), ('path', '/usr/bin/'), ('version', '3.2')] + >>> sorted(Interpreter.parse('/usr/bin/python3.2-dbg --foo --bar').items()) + [('debug', '-dbg'), ('name', 'python'), ('options', ('--foo', '--bar')),\ + ('path', '/usr/bin/'), ('version', '3.2')] + """ + result = SHEBANG_RE.search(shebang) + if not result: + return {} + result = result.groupdict() + if 'options' in result: + # TODO: do we need "--key value" here? + result['options'] = tuple(result['options'].split()) + if result['name'] == 'python' and result['version'] is None: + result['version'] = '2' + return result + + @classmethod + def from_file(cls, fpath): + """Read file's shebang and parse it.""" + interpreter = Interpreter() + with open(fpath, 'rb') as fp: + data = fp.read(96) + if b"\0" in data: + raise ValueError('cannot parse binary file') + # make sure only first line is checkeed + data = str(data, 'utf-8').split('\n')[0] + if not data.startswith('#!'): + raise ValueError("doesn't look like a shebang: %s" % data) + + parsed = cls.parse(data) + if not parsed: + raise ValueError("doesn't look like a shebang: %s" % data) + for key, val in parsed.items(): + setattr(interpreter, key, val) + return interpreter + + def sitedir(self, package=None, version=None, gdb=False): + """Return path to site-packages directory. + + Note that returned path is not the final location of .py files + + >>> i = Interpreter('python') + >>> i.sitedir(version='3.1') + '/usr/lib/python3/dist-packages/' + >>> i.sitedir(version='2.5') + '/usr/lib/python2.5/site-packages/' + >>> i.sitedir(version=Version('2.7')) + '/usr/lib/python2.7/dist-packages/' + >>> i.sitedir(version='3.1', gdb=True, package='python3-foo') + 'debian/python3-foo/usr/lib/debug/usr/lib/python3/dist-packages/' + >>> i.sitedir(version=Version('3.2')) + '/usr/lib/python3/dist-packages/' + """ + try: + version = Version(version or self.version) + except Exception as err: + raise ValueError("cannot find valid version: %s" % err) + if self.impl == 'pypy': + path = '/usr/lib/pypy/dist-packages/' + elif version << Version('2.6'): + path = "/usr/lib/python%s/site-packages/" % version + elif version << Version('3.0'): + path = "/usr/lib/python%s/dist-packages/" % version + else: + path = '/usr/lib/python3/dist-packages/' + + if gdb: + path = "/usr/lib/debug%s" % path + if package: + path = "debian/%s%s" % (package, path) + + return path + + def old_sitedirs(self, package=None, version=None, gdb=False): + """Return deprecated paths to site-packages directories.""" + try: + version = Version(version or self.version) + except Exception as err: + raise ValueError("cannot find valid version: %s" % err) + result = [] + for item in OLD_SITE_DIRS.get(self.impl, []): + if isinstance(item, str): + result.append(item.format(version)) + else: + res = item(version) + if res is not None: + result.append(res) + + if gdb: + result = ['/usr/lib/debug{}'.format(i) for i in result] + if self.impl.startswith('cpython'): + result.append('/usr/lib/debug/usr/lib/pyshared/python{}'.format(version)) + if package: + result = ['debian/{}{}'.format(package, i) for i in result] + + return result + + def parse_public_dir(self, path): + """Return version assigned to site-packages path + or True is it's unversioned public dir.""" + match = PUBLIC_DIR_RE[self.impl].match(path) + if match: + vers = match.groups(0) + if vers and vers[0]: + return Version(vers) + return True + + def should_ignore(self, path): + """Return True if path is used by another interpreter implementation.""" + cache_key = 'should_ignore_{}'.format(self.impl) + if cache_key not in self.__class__._cache: + expr = [v for k, v in INTERPRETER_DIR_TPLS.items() if k != self.impl] + regexp = re.compile('|'.join('({})'.format(i) for i in expr)) + self.__class__._cache[cache_key] = regexp + else: + regexp = self.__class__._cache[cache_key] + return regexp.search(path) + + def cache_file(self, fpath, version=None): + """Given path to a .py file, return path to its .pyc/.pyo file. + + This function is inspired by Python 3.2's imp.cache_from_source. + + :param fpath: path to file name + :param version: Python version + + >>> i = Interpreter('python') + >>> i.cache_file('foo.py', Version('3.1')) + 'foo.pyc' + >>> i.cache_file('bar/foo.py', '3.8') # doctest: +SKIP + 'bar/__pycache__/foo.cpython-38.pyc' + """ + version = Version(version or self.version) + last_char = 'o' if '-O' in self.options else 'c' + if version <= Version('3.1'): + return fpath + last_char + + fdir, fname = split(fpath) + if not fname.endswith('.py'): + fname += '.py' + return join(fdir, '__pycache__', "%s.%s.py%s" % + (fname[:-3], self.magic_tag(version), last_char)) + + def magic_number(self, version=None): + """Return magic number.""" + version = Version(version or self.version) + if self.impl == 'cpython2': + return '' + result = self._execute('import imp; print(imp.get_magic())', version) + return eval(result) + + def magic_tag(self, version=None): + """Return Python magic tag (used in __pycache__ dir to tag files). + + >>> i = Interpreter('python') + >>> i.magic_tag(version='3.8') # doctest: +SKIP + 'cpython-38' + """ + version = Version(version or self.version) + if self.impl.startswith('cpython') and version << Version('3.2'): + return '' + return self._execute('import imp; print(imp.get_tag())', version) + + def multiarch(self, version=None): + """Return multiarch tag.""" + version = Version(version or self.version) + try: + soabi, multiarch = self._get_config(version)[:2] + except Exception: + log.debug('cannot get multiarch', exc_info=True) + # interpreter without multiarch support + return '' + return multiarch + + def stableabi(self, version=None): + version = Version(version or self.version) + # stable ABI was introduced in Python 3.3 + if self.impl == 'cpython3' and version >> Version('3.2'): + return 'abi{}'.format(version.major) + + def soabi(self, version=None): + """Return SOABI flag (used to in .so files).""" + version = Version(version or self.version) + # NOTE: it's not the same as magic_tag + try: + soabi, multiarch = self._get_config(version)[:2] + except Exception: + log.debug('cannot get soabi', exc_info=True) + # interpreter without soabi support + return '' + return soabi + + @property + def include_dir(self): + """Return INCLUDE_DIR path. + + >>> Interpreter('python2.7').include_dir # doctest: +SKIP + '/usr/include/python2.7' + >>> Interpreter('python3.8-dbg').include_dir # doctest: +SKIP + '/usr/include/python3.8d' + """ + if self.impl == 'pypy': + return '/usr/lib/pypy/include' + try: + result = self._get_config()[2] + if result: + return result + except Exception: + result = '' + log.debug('cannot get include path', exc_info=True) + result = '/usr/include/{}'.format(self.name) + version = self.version + if self.debug: + if version >= '3.8': + result += 'd' + elif version << '3.3': + result += '_d' + else: + result += 'dm' + else: + if version >= '3.8': + pass + elif version >> '3.2': + result += 'm' + elif version == '3.2': + result += 'mu' + return result + + @property + def symlinked_include_dir(self): + """Return path to symlinked include directory.""" + if self.impl in ('cpython2', 'pypy') or self.debug \ + or self.version >> '3.7' or self.version << '3.3': + # these interpreters do not provide symlink, + # others provide it in libpython3.X-dev + return + try: + result = self._get_config()[2] + if result: + if result.endswith('m'): + return result[:-1] + else: + # there's include_dir, but no "m" + return + except Exception: + result = '/usr/include/{}'.format(self.name) + log.debug('cannot get include path', exc_info=True) + return result + + @property + def library_file(self): + """Return libfoo.so file path.""" + if self.impl == 'pypy': + return '' + libpl, ldlibrary = self._get_config()[3:5] + if ldlibrary.endswith('.a'): + # python3.1-dbg, python3.2, python3.2-dbg returned static lib + ldlibrary = ldlibrary.replace('.a', '.so') + if libpl and ldlibrary: + return join(libpl, ldlibrary) + raise Exception('cannot find library file for {}'.format(self)) + + def check_extname(self, fname, version=None): + """Return extension file name if file can be renamed.""" + if not version and not self.version: + return + + version = Version(version or self.version) + + if '/' in fname: + fdir, fname = fname.rsplit('/', 1) # in case full path was passed + else: + fdir = '' + + info = EXTFILE_RE.search(fname) + if not info: + return + info = info.groupdict() + if info['ver'] and (not version or version.minor is None): + # get version from soabi if version is not set of only major + # version number is set + version = Version("%s.%s" % (info['ver'][0], info['ver'][1])) + + if info['stableabi']: + # files with stable ABI in name don't need changes + return + if info['debug'] and self.debug is False: + # do not change Python 2.X extensions already marked as debug + # (the other way around is acceptable) + return + if info['soabi'] and info['multiarch']: + # already tagged, nothing we can do here + return + + try: + soabi, multiarch = self._get_config(version)[:2] + except Exception: + log.debug('cannot get soabi/multiarch', exc_info=True) + return + + if info['soabi'] and soabi and info['soabi'] != soabi: + return + + tmp_soabi = info['soabi'] or soabi + tmp_multiarch = info['multiarch'] or multiarch + + result = info['name'] + if result.endswith('module') and result != 'module' and ( + self.impl == 'cpython3' and version >> '3.2' or + self.impl == 'cpython2' and version == '2.7'): + result = result[:-6] + + if tmp_soabi: + result = "{}.{}".format(result, tmp_soabi) + if tmp_multiarch and not (self.impl == 'cpython3' and version << '3.3') and tmp_multiarch not in soabi: + result = "{}-{}".format(result, tmp_multiarch) + elif self.impl == 'cpython2' and version == '2.7' and tmp_multiarch: + result = "{}.{}".format(result, tmp_multiarch) + + if self.debug and self.impl == 'cpython2': + result += '_d' + result += '.so' + if fname == result: + return + return join(fdir, result) + + def suggest_pkg_name(self, name): + """Suggest binary package name with for given library name + + >>> Interpreter('python3.1').suggest_pkg_name('foo') + 'python3-foo' + >>> Interpreter('python3.8').suggest_pkg_name('foo_bar') + 'python3-foo-bar' + >>> Interpreter('python2.7-dbg').suggest_pkg_name('bar') + 'python-bar-dbg' + """ + name = name.replace('_', '-') + if self.impl == 'pypy': + return 'pypy-{}'.format(name) + version = '3' if self.impl == 'cpython3' else '' + result = 'python{}-{}'.format(version, name) + if self.debug: + result += '-dbg' + return result + + def _get_config(self, version=None): + version = Version(version or self.version) + # sysconfig module is available since Python 3.2 + # (also backported to Python 2.7) + if self.impl == 'pypy' or self.impl.startswith('cpython') and ( + version >> '2.6' and version << '3' + or version >> '3.1' or version == '3'): + cmd = 'import sysconfig as s;' + else: + cmd = 'from distutils import sysconfig as s;' + cmd += 'print("__SEP__".join(i or "" ' \ + 'for i in s.get_config_vars('\ + '"SOABI", "MULTIARCH", "INCLUDEPY", "LIBPL", "LDLIBRARY")))' + conf_vars = self._execute(cmd, version).split('__SEP__') + if conf_vars[1] in conf_vars[0]: + # Python >= 3.5 includes MILTIARCH in SOABI + conf_vars[0] = conf_vars[0].replace("-%s" % conf_vars[1], '') + try: + conf_vars[1] = os.environ['DEB_HOST_MULTIARCH'] + except KeyError: + pass + return conf_vars + + def _execute(self, command, version=None, cache=True): + version = Version(version or self.version) + exe = "{}{}".format(self.path, self._vstr(version)) + command = "{} -c '{}'".format(exe, command.replace("'", "\'")) + if cache and command in self.__class__._cache: + return self.__class__._cache[command] + if not exists(exe): + raise Exception("cannot execute command due to missing " + "interpreter: %s" % exe) + + output = execute(command) + if output['returncode'] != 0: + log.debug(output['stderr']) + raise Exception('{} failed with status code {}'.format(command, output['returncode'])) + + result = output['stdout'].splitlines() + + if len(result) == 1: + result = result[0] + + if cache: + self.__class__._cache[command] = result + + return result + +# due to circular imports issue +from dhpython.tools import execute +from dhpython.version import Version, default diff --git a/dhpython/markers.py b/dhpython/markers.py new file mode 100644 index 0000000..a0a3e55 --- /dev/null +++ b/dhpython/markers.py @@ -0,0 +1,70 @@ +# Copyright © 2022 Stefano Rivera <stefanor@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +""" +Handle Environment Markers +https://www.python.org/dev/peps/pep-0508/#environment-markers + +TODO: Ideally replace with the packaging library, but the API is currently +private: https://github.com/pypa/packaging/issues/496 +""" + +import re + + +SIMPLE_ENV_MARKER_RE = re.compile(r''' + (?P<marker>[a-z_]+) + \s* + (?P<op><=?|>=?|[=!~]=|===) + \s* + (?P<quote>['"]) + (?P<value>.*) # Could contain additional markers + (?P=quote) + ''', re.VERBOSE) +COMPLEX_ENV_MARKER_RE = re.compile(r''' + (?:\s|\)) + (?:and|or) + (?:\s|\() + ''', re.VERBOSE) + + +class ComplexEnvironmentMarker(Exception): + pass + + +def parse_environment_marker(marker): + """Parse a simple marker of <= 1 environment restriction""" + marker = marker.strip() + if marker.startswith('(') and marker.endswith(')'): + marker = marker[1:-1].strip() + + m = COMPLEX_ENV_MARKER_RE.search(marker) + if m: + raise ComplexEnvironmentMarker() + + m = SIMPLE_ENV_MARKER_RE.match(marker) + if not m: + raise ComplexEnvironmentMarker() + + return ( + m.group('marker'), + m.group('op'), + m.group('value'), + ) diff --git a/dhpython/option.py b/dhpython/option.py new file mode 100644 index 0000000..7d70dbe --- /dev/null +++ b/dhpython/option.py @@ -0,0 +1,30 @@ +# -*- coding: UTF-8 -*- +# Copyright © 2010-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import re + + +def compiled_regex(string): + """argparse regex type""" + try: + return re.compile(string) + except re.error: + raise ValueError("regular expression is not valid") diff --git a/dhpython/pydist.py b/dhpython/pydist.py new file mode 100644 index 0000000..015b5f2 --- /dev/null +++ b/dhpython/pydist.py @@ -0,0 +1,692 @@ +# Copyright © 2010-2020 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + +import email +import logging +import platform +import os +import re +from functools import partial +from os.path import exists, isdir, join +from subprocess import PIPE, Popen + +if __name__ == '__main__': + import sys + sys.path.append(os.path.abspath(join(os.path.dirname(__file__), '..'))) + +from dhpython import PKG_PREFIX_MAP, PUBLIC_DIR_RE,\ + PYDIST_DIRS, PYDIST_OVERRIDES_FNAMES, PYDIST_DPKG_SEARCH_TPLS +from dhpython.markers import ComplexEnvironmentMarker, parse_environment_marker +from dhpython.tools import memoize +from dhpython.version import get_requested_versions, Version + +log = logging.getLogger('dhpython') + +PYDIST_RE = re.compile(r""" + (?P<name>[A-Za-z][A-Za-z0-9_.-]*) # Python distribution name + \s* + (?P<vrange>(?:-?\d\.\d+(?:-(?:\d\.\d+)?)?)?) # version range + \s* + (?P<dependency>(?:[a-z][^;]*)?) # Debian dependency + (?: # optional upstream version -> Debian version translator + ;\s* + (?P<standard>PEP386)? # PEP-386 mode + \s* + (?P<rules>(?:s|tr|y).*)? # translator rules + )? + """, re.VERBOSE) +REQUIRES_RE = re.compile(r''' + (?P<name>[A-Za-z][A-Za-z0-9_.-]*) # Python distribution name + \s* + (?P<enabled_extras>(?:\[[^\]]*\])?) # ignored for now + \s* + \(? # optional parenthesis + (?: # optional minimum/maximum version + (?P<operator><=?|>=?|==|!=|~=) + \s* + (?P<version>(\w|[-.*])+) + (?: # optional interval minimum/maximum version + \s* + , + \s* + (?P<operator2><=?|>=?|==|!=) + \s* + (?P<version2>(\w|[-.])+) + )? + )? + \)? # optional closing parenthesis + \s* + (?:; # optional environment markers + (?P<environment_marker>.+) + )? + ''', re.VERBOSE) +EXTRA_RE = re.compile(r''' + ; + \s* + extra + \s* + == + \s* + (?P<quote>['"]) + (?P<section>[a-zA-Z0-9-_.]+) + (?P=quote) + ''', re.VERBOSE) +REQ_SECTIONS_RE = re.compile(r''' + ^ + \[ + (?P<section>[a-zA-Z0-9-_.]+)? + \s* + (?:: + (?P<environment_marker>.+) + )? + \] + \s* + $ + ''', re.VERBOSE) +DEB_VERS_OPS = { + '==': '=', + '<': '<<', + '>': '>>', + '~=': '>=', +} + + +def validate(fpath): + """Check if pydist file looks good.""" + with open(fpath, encoding='utf-8') as fp: + for line in fp: + line = line.strip() + if line.startswith('#') or not line: + continue + if not PYDIST_RE.match(line): + log.error('invalid pydist data in file %s: %s', + fpath.rsplit('/', 1)[-1], line) + return False + return True + + +@memoize +def load(impl): + """Load information about installed Python distributions. + + :param impl: interpreter implementation, f.e. cpython2, cpython3, pypy + :type impl: str + """ + fname = PYDIST_OVERRIDES_FNAMES.get(impl) + if exists(fname): + to_check = [fname] # first one! + else: + to_check = [] + + dname = PYDIST_DIRS.get(impl) + if isdir(dname): + to_check.extend(join(dname, i) for i in os.listdir(dname)) + + fbdir = os.environ.get('DH_PYTHON_DIST', '/usr/share/dh-python/dist/') + fbname = join(fbdir, '{}_fallback'.format(impl)) + if exists(fbname): # fall back generated at dh-python build time + to_check.append(fbname) # last one! + + result = {} + for fpath in to_check: + with open(fpath, encoding='utf-8') as fp: + for line in fp: + line = line.strip() + if line.startswith('#') or not line: + continue + dist = PYDIST_RE.search(line) + if not dist: + raise Exception('invalid pydist line: %s (in %s)' % (line, fpath)) + dist = dist.groupdict() + name = safe_name(dist['name']) + dist['versions'] = get_requested_versions(impl, dist['vrange']) + dist['dependency'] = dist['dependency'].strip() + if dist['rules']: + dist['rules'] = dist['rules'].split(';') + else: + dist['rules'] = [] + result.setdefault(name, []).append(dist) + return result + + +def guess_dependency(impl, req, version=None, bdep=None, + accept_upstream_versions=False): + bdep = bdep or {} + log.debug('trying to find dependency for %s (python=%s)', + req, version) + if isinstance(version, str): + version = Version(version) + + # some upstreams have weird ideas for distribution name... + name, rest = re.compile('([^!><=~ \(\)\[;]+)(.*)').match(req).groups() + # TODO: check stdlib and dist-packaged for name.py and name.so files + req = safe_name(name) + rest + + data = load(impl) + req_d = REQUIRES_RE.match(req) + if not req_d: + log.info('please ask dh_python3 author to fix REQUIRES_RE ' + 'or your upstream author to fix requires.txt') + raise Exception('requirement is not valid: %s' % req) + req_d = req_d.groupdict() + + env_marker_alts = '' + if req_d['environment_marker']: + action = check_environment_marker_restrictions( + req, + req_d['environment_marker'], + impl) + if action is False: + return + elif action is True: + pass + else: + env_marker_alts = ' ' + action + + name = req_d['name'] + details = data.get(name.lower()) + if details: + log.debug("dependency: module seems to be installed") + for item in details: + if version and version not in item.get('versions', version): + # rule doesn't match version, try next one + continue + if not item['dependency']: + log.debug("dependency: requirement ignored") + return # this requirement should be ignored + if item['dependency'].endswith(')'): + # no need to translate versions if version is hardcoded in + # Debian dependency + log.debug("dependency: requirement already has hardcoded version") + return item['dependency'] + env_marker_alts + if req_d['operator'] == '==' and req_d['version'].endswith('*'): + # Translate "== 1.*" to "~= 1.0" + req_d['operator'] = '~=' + req_d['version'] = req_d['version'].replace('*', '0') + log.debug("dependency: translated wildcard version to semver limit") + if req_d['version'] and (item['standard'] or item['rules']) and\ + req_d['operator'] not in (None, '!='): + o = _translate_op(req_d['operator']) + v = _translate(req_d['version'], item['rules'], item['standard']) + d = "%s (%s %s)%s" % ( + item['dependency'], o, v, env_marker_alts) + if req_d['version2'] and req_d['operator2'] not in (None,'!='): + o2 = _translate_op(req_d['operator2']) + v2 = _translate(req_d['version2'], item['rules'], item['standard']) + d += ", %s (%s %s)%s" % ( + item['dependency'], o2, v2, env_marker_alts) + elif req_d['operator'] == '~=': + o2 = '<<' + v2 = _translate(_max_compatible(req_d['version']), item['rules'], item['standard']) + d += ", %s (%s %s)%s" % ( + item['dependency'], o2, v2, env_marker_alts) + log.debug("dependency: constructed version") + return d + elif accept_upstream_versions and req_d['version'] and \ + req_d['operator'] not in (None,'!='): + o = _translate_op(req_d['operator']) + d = "%s (%s %s)%s" % ( + item['dependency'], o, req_d['version'], env_marker_alts) + if req_d['version2'] and req_d['operator2'] not in (None,'!='): + o2 = _translate_op(req_d['operator2']) + d += ", %s (%s %s)%s" % ( + item['dependency'], o2, req_d['version2'], + env_marker_alts) + elif req_d['operator'] == '~=': + o2 = '<<' + d += ", %s (%s %s)%s" % ( + item['dependency'], o2, + _max_compatible(req_d['version']), env_marker_alts) + log.debug("dependency: constructed upstream version") + return d + else: + if item['dependency'] in bdep: + if None in bdep[item['dependency']] and bdep[item['dependency']][None]: + log.debug("dependency: included in build-deps with limits ") + return "{} ({}){}".format( + item['dependency'], bdep[item['dependency']][None], + env_marker_alts) + # if arch in bdep[item['dependency']]: + # TODO: handle architecture specific dependencies from build depends + # (current architecture is needed here) + log.debug("dependency: included in build-deps") + return item['dependency'] + env_marker_alts + + # search for Egg metadata file or directory (using dpkg -S) + dpkg_query_tpl, regex_filter = PYDIST_DPKG_SEARCH_TPLS[impl] + dpkg_query = dpkg_query_tpl.format(ci_regexp(safe_name(name))) + + log.debug("invoking dpkg -S %s", dpkg_query) + process = Popen(('/usr/bin/dpkg', '-S', dpkg_query), + stdout=PIPE, stderr=PIPE) + stdout, stderr = process.communicate() + if process.returncode == 0: + result = set() + stdout = str(stdout, 'utf-8') + for line in stdout.split('\n'): + if not line.strip(): + continue + pkg, path = line.split(':', 1) + if regex_filter and not re.search(regex_filter, path): + continue + result.add(pkg) + if len(result) > 1: + log.error('more than one package name found for %s dist', name) + elif not result: + log.debug('dpkg -S did not find package for %s', name) + else: + log.debug('dependency: found a result with dpkg -S') + return result.pop() + env_marker_alts + else: + log.debug('dpkg -S did not find package for %s: %s', name, stderr) + + pname = sensible_pname(impl, name) + log.info('Cannot find package that provides %s. ' + 'Please add package that provides it to Build-Depends or ' + 'add "%s %s" line to %s or add proper ' + 'dependency to Depends by hand and ignore this info.', + name, safe_name(name), pname, PYDIST_OVERRIDES_FNAMES[impl]) + # return pname + + +def check_environment_marker_restrictions(req, marker_str, impl): + """Check wither we should include or skip a dependency based on its + environment markers. + + Returns: True - to keep a dependency + False - to skip it + str - to append "| foo" to generated dependencies + """ + if impl != 'cpython3': + log.info('Ignoring environment markers for non-Python 3.x: %s', req) + return False + + try: + marker, op, value = parse_environment_marker(marker_str) + except ComplexEnvironmentMarker: + log.info('Ignoring complex environment marker: %s', req) + return False + + # TODO: Use dynamic values when building arch-dependent + # binaries, otherwise static values + # TODO: Hurd values? + supported_values = { + 'implementation_name': ('cpython', 'pypy'), + 'os_name': ('posix',), + 'platform_system': ('GNU/kFreeBSD', 'Linux'), + 'platform_machine': (platform.machine(),), + 'platform_python_implementation': ('CPython', 'PyPy'), + 'sys_platform': ( + 'gnukfreebsd8', 'gnukfreebsd9', 'gnukfreebsd10', + 'gnukfreebsd11', 'gnukfreebsd12', 'gnukfreebsd13', + 'linux'), + } + if marker in supported_values: + sv = supported_values[marker] + if op in ('==', '!='): + if ((op == '==' and value not in sv) + or (op == '!=' and value in sv)): + log.debug('Skipping requirement (%s != %s): %s', + value, sv, req) + return False + else: + log.info( + 'Skipping requirement with unhandled environment marker ' + 'comparison: %s', req) + return False + + elif marker in ('python_version', 'python_full_version', + 'implementation_version'): + # TODO: Replace with full PEP-440 parser + env_ver = value + split_ver = value.split('.') + if marker == 'python_version': + version_parts = 2 + elif marker == 'python_full_version': + version_parts = 3 + else: + version_parts = len(split_ver) + + if '*' in env_ver: + if split_ver.index('*') != len(split_ver) -1: + log.info('Skipping requirement with intermediate wildcard: %s', + req) + return False + split_ver.pop() + env_ver = '.'.join(split_ver) + if op == '==': + if marker == 'python_full_version': + marker = 'python_version' + version_parts = 2 + else: + op == '=~' + elif op == '!=': + if marker == 'python_full_version': + marker = 'python_version' + version_parts = 2 + else: + log.info('Ignoring wildcard != requirement, not ' + 'representable in Debian: %s', req) + return True + else: + log.info('Skipping requirement with %s on a wildcard: %s', + op, req) + return False + + int_ver = [] + for ver_part in split_ver: + if ver_part.isdigit(): + int_ver.append(int(ver_part)) + else: + env_ver = '.'.join(str(x) for x in int_ver) + log.info('Truncating unparseable version %s to %s in %s', + value, env_ver, req) + break + + if len(int_ver) < version_parts: + int_ver.append(0) + env_ver += '.0' + next_ver = int_ver.copy() + next_ver[version_parts - 1] += 1 + next_ver = '.'.join(str(x) for x in next_ver) + prev_ver = int_ver.copy() + prev_ver[version_parts - 1] -= 1 + prev_ver = '.'.join(str(x) for x in prev_ver) + + if op == '<': + if int_ver <= [3, 0, 0]: + return False + return '| python3 (>> {})'.format(env_ver) + elif op == '<=': + return '| python3 (>> {})'.format(next_ver) + elif op == '>=': + if int_ver < [3, 0, 0]: + return True + return '| python3 (<< {})'.format(env_ver) + elif op == '>': + if int_ver < [3, 0, 0]: + return True + return '| python3 (<< {})'.format(next_ver) + elif op in ('==', '==='): + # === is arbitrary equality (PEP 440) + if marker == 'python_version' or op == '==': + return '| python3 (<< {}) | python3 (>> {})'.format( + env_ver, next_ver) + else: + log.info( + 'Skipping requirement with %s environment marker, cannot ' + 'model in Debian deps: %s', op, req) + return False + elif op == '~=': # Compatible equality (PEP 440) + ceq_next_ver = int_ver[:2] + ceq_next_ver[1] += 1 + ceq_next_ver = '.'.join(str(x) for x in ceq_next_ver) + return '| python3 (<< {}) | python3 (>> {})'.format( + env_ver, ceq_next_ver) + elif op == '!=': + log.info('Ignoring != comparison in environment marker, cannot ' + 'model in Debian deps: %s', req) + return True + + elif marker == 'extra': + # Handled in section logic of parse_requires_dist() + return True + else: + log.info('Skipping requirement with unknown environment marker: %s', + marker) + return False + return True + + +def parse_pydep(impl, fname, bdep=None, options=None, + depends_sec=None, recommends_sec=None, suggests_sec=None): + depends_sec = depends_sec or [] + recommends_sec = recommends_sec or [] + suggests_sec = suggests_sec or [] + + public_dir = PUBLIC_DIR_RE[impl].match(fname) + ver = None + if public_dir and public_dir.groups() and len(public_dir.group(1)) != 1: + ver = public_dir.group(1) + + guess_deps = partial(guess_dependency, impl=impl, version=ver, bdep=bdep, + accept_upstream_versions=getattr( + options, 'accept_upstream_versions', False)) + + result = {'depends': [], 'recommends': [], 'suggests': []} + modified = section = False + env_action = True + processed = [] + with open(fname, 'r', encoding='utf-8') as fp: + for line in fp: + line = line.strip() + if not line or line.startswith('#'): + processed.append(line) + continue + if line.startswith('['): + m = REQ_SECTIONS_RE.match(line) + if not m: + log.info('Skipping section %s, unable to parse header', + line) + processed.append(line) + section = object() + continue + section = m.group('section') + env_action = True + if m.group('environment_marker'): + env_action = check_environment_marker_restrictions( + line, + m.group('environment_marker'), + impl) + processed.append(line) + continue + if section: + if section in depends_sec: + result_key = 'depends' + elif section in recommends_sec: + result_key = 'recommends' + elif section in suggests_sec: + result_key = 'suggests' + else: + processed.append(line) + continue + else: + result_key = 'depends' + + dependency = None + if env_action: + dependency = guess_deps(req=line) + if dependency and isinstance(env_action, str): + dependency = ', '.join( + part.strip() + ' ' + env_action + for part in dependency.split(',')) + + if dependency: + result[result_key].append(dependency) + modified = True + else: + processed.append(line) + if modified and public_dir: + with open(fname, 'w', encoding='utf-8') as fp: + fp.writelines(i + '\n' for i in processed) + return result + + +def parse_requires_dist(impl, fname, bdep=None, options=None, depends_sec=None, + recommends_sec=None, suggests_sec=None): + """Extract dependencies from a dist-info/METADATA file""" + depends_sec = depends_sec or [] + recommends_sec = recommends_sec or [] + suggests_sec = suggests_sec or [] + + public_dir = PUBLIC_DIR_RE[impl].match(fname) + ver = None + if public_dir and public_dir.groups() and len(public_dir.group(1)) != 1: + ver = public_dir.group(1) + + guess_deps = partial(guess_dependency, impl=impl, version=ver, bdep=bdep, + accept_upstream_versions=getattr( + options, 'accept_upstream_versions', False)) + result = {'depends': [], 'recommends': [], 'suggests': []} + section = None + with open(fname, 'r', encoding='utf-8') as fp: + metadata = email.message_from_string(fp.read()) + requires = metadata.get_all('Requires-Dist', []) + for req in requires: + m = EXTRA_RE.search(req) + result_key = 'depends' + if m: + section = m.group('section') + if section: + if section in depends_sec: + result_key = 'depends' + elif section in recommends_sec: + result_key = 'recommends' + elif section in suggests_sec: + result_key = 'suggests' + else: + continue + dependency = guess_deps(req=req) + if dependency: + result[result_key].append(dependency) + return result + + +def safe_name(name): + """Emulate distribute's safe_name.""" + return re.compile('[^A-Za-z0-9.]+').sub('_', name).lower() + + +def sensible_pname(impl, egg_name): + """Guess Debian package name from Egg name.""" + egg_name = safe_name(egg_name).replace('_', '-') + if egg_name.startswith('python-'): + egg_name = egg_name[7:] + return '{}-{}'.format(PKG_PREFIX_MAP[impl], egg_name.lower()) + + +def ci_regexp(name): + """Return case insensitive dpkg -S regexp.""" + return ''.join("[%s%s]" % (i.upper(), i) if i.isalpha() else i for i in name.lower()) + + +PRE_VER_RE = re.compile(r'[-.]?(alpha|beta|rc|dev|a|b|c)') +GROUP_RE = re.compile(r'\$(\d+)') + + +def _pl2py(pattern): + """Convert Perl RE patterns used in uscan to Python's + + >>> print(_pl2py('foo$3')) + foo\g<3> + """ + return GROUP_RE.sub(r'\\g<\1>', pattern) + + +def _max_compatible(version): + """Return the maximum version compatible with `version` in PEP440 terms, + used by ~= requires version specifiers. + + https://www.python.org/dev/peps/pep-0440/#compatible-release + + >>> _max_compatible('2.2') + '3' + >>> _max_compatible('1.4.5') + '1.5' + >>> _max_compatible('1.3.alpha4') + '2' + >>> _max_compatible('2.1.3.post5') + '2.2' + + """ + v = Version(version) + v.serial = None + v.releaselevel = None + if v.micro is not None: + v.micro = None + return str(v + 1) + v.minor = None + return str(v + 1) + + +def _translate(version, rules, standard): + """Translate Python version into Debian one. + + >>> _translate('1.C2betac', ['s/c//gi'], None) + '1.2beta' + >>> _translate('5-fooa1.2beta3-fooD', + ... ['s/^/1:/', 's/-foo//g', 's:([A-Z]):+$1:'], 'PEP386') + '1:5~a1.2~beta3+D' + >>> _translate('x.y.x.z', ['tr/xy/ab/', 'y,z,Z,'], None) + 'a.b.a.Z' + """ + for rule in rules: + # uscan supports s, tr and y operations + if rule.startswith(('tr', 'y')): + # Note: no support for escaped separator in the pattern + pos = 1 if rule.startswith('y') else 2 + tmp = rule[pos + 1:].split(rule[pos]) + version = version.translate(str.maketrans(tmp[0], tmp[1])) + elif rule.startswith('s'): + # uscan supports: g, u and x flags + tmp = rule[2:].split(rule[1]) + pattern = re.compile(tmp[0]) + count = 1 + if tmp[2:]: + flags = tmp[2] + if 'g' in flags: + count = 0 + if 'i' in flags: + pattern = re.compile(tmp[0], re.I) + version = pattern.sub(_pl2py(tmp[1]), version, count) + else: + log.warn('unknown rule ignored: %s', rule) + if standard == 'PEP386': + version = PRE_VER_RE.sub(r'~\g<1>', version) + return version + + +def _translate_op(operator): + """Translate Python version operator into Debian one. + + >>> _translate_op('==') + '=' + >>> _translate_op('<') + '<<' + >>> _translate_op('<=') + '<=' + """ + return DEB_VERS_OPS.get(operator, operator) + + +if __name__ == '__main__': + impl = os.environ.get('IMPL', 'cpython3') + for i in sys.argv[1:]: + if os.path.isfile(i): + try: + print(', '.join(parse_pydep(impl, i)['depends'])) + except Exception as err: + log.error('%s: cannot guess (%s)', i, err) + else: + try: + print(guess_dependency(impl, i) or '') + except Exception as err: + log.error('%s: cannot guess (%s)', i, err) diff --git a/dhpython/tools.py b/dhpython/tools.py new file mode 100644 index 0000000..512f944 --- /dev/null +++ b/dhpython/tools.py @@ -0,0 +1,340 @@ +# -*- coding: UTF-8 -*- +# Copyright © 2010-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +import os +import re +import locale +from datetime import datetime +from glob import glob +from pickle import dumps +from shutil import rmtree +from os.path import exists, getsize, isdir, islink, join, split +from subprocess import Popen, PIPE + +log = logging.getLogger('dhpython') +EGGnPTH_RE = re.compile(r'(.*?)(-py\d\.\d(?:-[^.]*)?)?(\.egg-info|\.pth)$') +SHAREDLIB_RE = re.compile(r'NEEDED.*libpython(\d\.\d)') + + +def relpath(target, link): + """Return relative path. + + >>> relpath('/usr/share/python-foo/foo.py', '/usr/bin/foo', ) + '../share/python-foo/foo.py' + """ + t = target.split('/') + l = link.split('/') + while l and l[0] == t[0]: + del l[0], t[0] + return '/'.join(['..'] * (len(l) - 1) + t) + + +def relative_symlink(target, link): + """Create relative symlink.""" + return os.symlink(relpath(target, link), link) + + +def move_file(fpath, dstdir): + """Move file to dstdir. Works with symlinks (including relative ones).""" + if isdir(fpath): + dname = split(fpath)[-1] + for fn in os.listdir(fpath): + move_file(join(fpath, fn), join(dstdir, dname)) + + if islink(fpath): + dstpath = join(dstdir, split(fpath)[-1]) + relative_symlink(os.readlink(fpath), dstpath) + os.remove(fpath) + else: + os.rename(fpath, dstdir) + + +def move_matching_files(src, dst, pattern, sub=None, repl=''): + """Move files (preserving path) that match given pattern. + + move_matching_files('foo/bar/', 'foo/baz/', 'spam/.*\.so$') + will move foo/bar/a/b/c/spam/file.so to foo/baz/a/b/c/spam/file.so + + :param sub: regular expression for path part that will be replaced with `repl` + :param repl: replacement for `sub` + """ + match = re.compile(pattern).search + if sub: + sub = re.compile(sub).sub + repl = repl or '' + for root, dirs, filenames in os.walk(src): + for fn in filenames: + spath = join(root, fn) + if match(spath): + if sub is not None: + spath = sub(repl, spath) + dpath = join(dst, relpath(spath, src)) + os.renames(spath, dpath) + + +def fix_shebang(fpath, replacement=None): + """Normalize file's shebang. + + :param replacement: new shebang command (path to interpreter and options) + """ + try: + interpreter = Interpreter.from_file(fpath) + except Exception as err: + log.debug('fix_shebang (%s): %s', fpath, err) + return None + + if not replacement and interpreter.version == '2': + # we'll drop /usr/bin/python symlink from python package at some point + replacement = '/usr/bin/python2' + if interpreter.debug: + replacement += '-dbg' + elif not replacement and interpreter.path != '/usr/bin/': # f.e. /usr/local/* or */bin/env + interpreter.path = '/usr/bin' + replacement = repr(interpreter) + if replacement: + log.info('replacing shebang in %s', fpath) + try: + with open(fpath, 'rb') as fp: + fcontent = fp.readlines() + except IOError: + log.error('cannot open %s', fpath) + return False + # do not catch IOError here, the file is zeroed at this stage so it's + # better to fail + with open(fpath, 'wb') as fp: + fp.write(("#! %s\n" % replacement).encode('utf-8')) + fp.writelines(fcontent[1:]) + return True + + +def so2pyver(fpath): + """Return libpython version file is linked to or None. + + :rtype: tuple + :returns: Python version + """ + + cmd = "readelf -Wd '%s'" % fpath + process = Popen(cmd, stdout=PIPE, shell=True) + encoding = locale.getdefaultlocale()[1] or 'utf-8' + match = SHAREDLIB_RE.search(str(process.stdout.read(), encoding=encoding)) + if match: + return Version(match.groups()[0]) + + +def clean_egg_name(name): + """Remove Python version and platform name from Egg files/dirs. + + >>> clean_egg_name('python_pipeline-0.1.3_py3k-py3.1.egg-info') + 'python_pipeline-0.1.3_py3k.egg-info' + >>> clean_egg_name('Foo-1.2-py2.7-linux-x86_64.egg-info') + 'Foo-1.2.egg-info' + """ + match = EGGnPTH_RE.match(name) + if match and match.group(2) is not None: + return ''.join(match.group(1, 3)) + return name + + +def parse_ns(fpaths, other=None): + """Parse namespace_packages.txt files.""" + result = set(other or []) + for fpath in fpaths: + with open(fpath, 'r', encoding='utf-8') as fp: + for line in fp: + if line: + result.add(line.strip()) + return result + + +def remove_ns(interpreter, package, namespaces, versions): + """Remove empty __init__.py files for requested namespaces.""" + if not isinstance(namespaces, set): + namespaces = set(namespaces) + keep = set() + for ns in namespaces: + for version in versions: + fpath = join(interpreter.sitedir(package, version), *ns.split('.')) + fpath = join(fpath, '__init__.py') + if not exists(fpath): + continue + if getsize(fpath) != 0: + log.warning('file not empty, cannot share %s namespace', ns) + keep.add(ns) + break + + # return a set of namespaces that should be handled by pycompile/pyclean + result = namespaces - keep + + # remove empty __init__.py files, if available + for ns in result: + for version in versions: + dpath = join(interpreter.sitedir(package, version), *ns.split('.')) + fpath = join(dpath, '__init__.py') + if exists(fpath): + os.remove(fpath) + if not os.listdir(dpath): + os.rmdir(dpath) + # clean pyshared dir as well + dpath = join('debian', package, 'usr/share/pyshared', *ns.split('.')) + fpath = join(dpath, '__init__.py') + if exists(fpath): + os.remove(fpath) + if not os.listdir(dpath): + os.rmdir(dpath) + return result + + +def execute(command, cwd=None, env=None, log_output=None, shell=True): + """Execute external shell command. + + :param cdw: current working directory + :param env: environment + :param log_output: + * opened log file or path to this file, or + * None if output should be included in the returned dict, or + * False if output should be redirected to stdout/stderr + """ + args = {'shell': shell, 'cwd': cwd, 'env': env} + close = False + if log_output is False: + pass + elif log_output is None: + args.update(stdout=PIPE, stderr=PIPE) + elif log_output: + if isinstance(log_output, str): + close = True + log_output = open(log_output, 'a', encoding='utf-8') + log_output.write('\n# command executed on {}'.format(datetime.now().isoformat())) + log_output.write('\n$ {}\n'.format(command)) + log_output.flush() + args.update(stdout=log_output, stderr=log_output) + + log.debug('invoking: %s', command) + with Popen(command, **args) as process: + stdout, stderr = process.communicate() + close and log_output.close() + return dict(returncode=process.returncode, + stdout=stdout and str(stdout, 'utf-8'), + stderr=stderr and str(stderr, 'utf-8')) + + +class memoize: + def __init__(self, func): + self.func = func + self.cache = {} + + def __call__(self, *args, **kwargs): + key = dumps((args, kwargs)) + if key not in self.cache: + self.cache[key] = self.func(*args, **kwargs) + return self.cache[key] + + +def pyinstall(interpreter, package, vrange): + """Install local files listed in pkg.pyinstall files as public modules.""" + srcfpath = "./debian/%s.pyinstall" % package + if not exists(srcfpath): + return + impl = interpreter.impl + versions = get_requested_versions(impl, vrange) + + for line in open(srcfpath, encoding='utf-8'): + if not line or line.startswith('#'): + continue + details = INSTALL_RE.match(line) + if not details: + raise ValueError("unrecognized line: %s" % line) + details = details.groupdict() + if details['module']: + details['module'] = details['module'].replace('.', '/') + myvers = versions & get_requested_versions(impl, details['vrange']) + if not myvers: + log.debug('%s.pyinstall: no matching versions for line %s', + package, line) + continue + files = glob(details['pattern']) + if not files: + raise ValueError("missing file(s): %s" % details['pattern']) + for fpath in files: + fpath = fpath.lstrip('/.') + if details['module']: + dstname = join(details['module'], split(fpath)[1]) + elif fpath.startswith('debian/'): + dstname = fpath[7:] + else: + dstname = fpath + for version in myvers: + dstfpath = join(interpreter.sitedir(package, version), dstname) + dstdir = split(dstfpath)[0] + if not exists(dstdir): + os.makedirs(dstdir) + if exists(dstfpath): + os.remove(dstfpath) + os.link(fpath, dstfpath) + + +def pyremove(interpreter, package, vrange): + """Remove public modules listed in pkg.pyremove file.""" + srcfpath = "./debian/%s.pyremove" % package + if not exists(srcfpath): + return + impl = interpreter.impl + versions = get_requested_versions(impl, vrange) + + for line in open(srcfpath, encoding='utf-8'): + if not line or line.startswith('#'): + continue + details = REMOVE_RE.match(line) + if not details: + raise ValueError("unrecognized line: %s: %s" % (package, line)) + details = details.groupdict() + myvers = versions & get_requested_versions(impl, details['vrange']) + if not myvers: + log.debug('%s.pyremove: no matching versions for line %s', + package, line) + for version in myvers: + site_dirs = interpreter.old_sitedirs(package, version) + site_dirs.append(interpreter.sitedir(package, version)) + for sdir in site_dirs: + files = glob(sdir + '/' + details['pattern']) + for fpath in files: + if isdir(fpath): + rmtree(fpath) + else: + os.remove(fpath) + +from dhpython.interpreter import Interpreter +from dhpython.version import Version, get_requested_versions, RANGE_PATTERN +INSTALL_RE = re.compile(r""" + (?P<pattern>.+?) # file pattern + (?:\s+ # optional Python module name: + (?P<module>[A-Za-z][A-Za-z0-9_.]*)? + )? + \s* # optional version range: + (?P<vrange>%s)?$ +""" % RANGE_PATTERN, re.VERBOSE) +REMOVE_RE = re.compile(r""" + (?P<pattern>.+?) # file pattern + \s* # optional version range: + (?P<vrange>%s)?$ +""" % RANGE_PATTERN, re.VERBOSE) diff --git a/dhpython/version.py b/dhpython/version.py new file mode 100644 index 0000000..98c16b7 --- /dev/null +++ b/dhpython/version.py @@ -0,0 +1,457 @@ +# Copyright © 2010-2013 Piotr Ożarowski <piotr@debian.org> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +import re +from os.path import exists + +from dhpython import _defaults + +RANGE_PATTERN = r'(-)?(\d\.\d+)(?:(-)(\d\.\d+)?)?' +RANGE_RE = re.compile(RANGE_PATTERN) +VERSION_RE = re.compile(r''' + (?P<major>\d+)\.? + (?P<minor>\d+)?\.? + (?P<micro>\d+)?[.\s]? + (?P<releaselevel>alpha|beta|candidate|final)?[.\s]? + (?P<serial>\d+)?''', re.VERBOSE) + +log = logging.getLogger('dhpython') +Interpreter = None + + +class Version: + # TODO: Upgrade to PEP-440 + def __init__(self, value=None, major=None, minor=None, micro=None, + releaselevel=None, serial=None): + """Construct a new instance. + + >>> Version(major=0, minor=0, micro=0, releaselevel=0, serial=0) + Version('0.0') + >>> Version('0.0') + Version('0.0') + """ + if isinstance(value, (tuple, list)): + value = '.'.join(str(i) for i in value) + if isinstance(value, Version): + for name in ('major', 'minor', 'micro', 'releaselevel', 'serial'): + setattr(self, name, getattr(value, name)) + return + comp = locals() + del comp['self'] + del comp['value'] + if value: + match = VERSION_RE.match(value) + for name, value in match.groupdict().items() if match else []: + if value is not None and comp[name] is None: + comp[name] = value + for name, value in comp.items(): + if name != 'releaselevel' and value is not None: + value = int(value) + setattr(self, name, value) + if self.major is None: + raise ValueError('major component is required') + + def __str__(self): + """Return major.minor or major string. + + >>> str(Version(major=3, minor=2, micro=1, releaselevel='final', serial=4)) + '3.2' + >>> str(Version(major=2)) + '2' + """ + result = str(self.major) + if self.minor is not None: + result += '.{}'.format(self.minor) + return result + + def __hash__(self): + return hash(repr(self)) + + def __repr__(self): + """Return full version string. + + >>> repr(Version(major=3, minor=2, micro=1, releaselevel='final', serial=4)) + "Version('3.2.1.final.4')" + >>> repr(Version(major=2)) + "Version('2')" + """ + result = "Version('{}".format(self) + for name in ('micro', 'releaselevel', 'serial'): + value = getattr(self, name) + if not value: + break + result += '.{}'.format(value) + return result + "')" + + def __add__(self, other): + """Return next version. + + >>> Version('3.1') + 1 + Version('3.2') + >>> Version('2') + '1' + Version('3') + """ + result = Version(self) + if self.minor is None: + result.major += int(other) + else: + result.minor += int(other) + return result + + def __sub__(self, other): + """Return previous version. + + >>> Version('3.1') - 1 + Version('3.0') + >>> Version('3') - '1' + Version('2') + """ + result = Version(self) + if self.minor is None: + result.major -= int(other) + new = result.major + else: + result.minor -= int(other) + new = result.minor + if new < 0: + raise ValueError('cannot decrease version further') + return result + + def __eq__(self, other): + try: + other = Version(other) + except Exception: + return False + return self.__cmp(other) == 0 + + def __lt__(self, other): + return self.__cmp(other) < 0 + + def __le__(self, other): + return self.__cmp(other) <= 0 + + def __gt__(self, other): + return self.__cmp(other) > 0 + + def __ge__(self, other): + return self.__cmp(other) >= 0 + + def __lshift__(self, other): + """Compare major.minor or major only (if minor is not set). + + >>> Version('2.6') << Version('2.7') + True + >>> Version('2.6') << Version('2.6.6') + False + >>> Version('3') << Version('2') + False + >>> Version('3.1') << Version('2') + False + >>> Version('2') << Version('3.2.1.alpha.3') + True + """ + if not isinstance(other, Version): + other = Version(other) + if self.minor is None or other.minor is None: + return self.__cmp(other, ignore='minor') < 0 + else: + return self.__cmp(other, ignore='micro') < 0 + + def __rshift__(self, other): + """Compare major.minor or major only (if minor is not set). + + >>> Version('2.6') >> Version('2.7') + False + >>> Version('2.6.7') >> Version('2.6.6') + False + >>> Version('3') >> Version('2') + True + >>> Version('3.1') >> Version('2') + True + >>> Version('2.1') >> Version('3.2.1.alpha.3') + False + """ + if not isinstance(other, Version): + other = Version(other) + if self.minor is None or other.minor is None: + return self.__cmp(other, ignore='minor') > 0 + else: + return self.__cmp(other, ignore='micro') > 0 + + def __cmp(self, other, ignore=None): + if not isinstance(other, Version): + other = Version(other) + for name in ('major', 'minor', 'micro', 'releaselevel', 'serial'): + if name == ignore: + break + value1 = getattr(self, name) or 0 + value2 = getattr(other, name) or 0 + if name == 'releaselevel': + rmap = {'alpha': -3, 'beta': -2, 'candidate': -1, 'final': 0} + value1 = rmap.get(value1, 0) + value2 = rmap.get(value2, 0) + if value1 == value2: + continue + return (value1 > value2) - (value1 < value2) + return 0 + + +class VersionRange: + def __init__(self, value=None, minver=None, maxver=None): + if minver: + self.minver = Version(minver) + else: + self.minver = None + if maxver: + self.maxver = Version(maxver) + else: + self.maxver = None + + if value: + minver, maxver = self.parse(value) + if minver and self.minver is None: + self.minver = minver + if maxver and self.maxver is None: + self.maxver = maxver + + def __bool__(self): + if self.minver is not None or self.maxver is not None: + return True + return False + + def __str__(self): + """Return version range string from given range. + + >>> str(VersionRange(minver='3.4')) + '3.4-' + >>> str(VersionRange(minver='3.4', maxver='3.6')) + '3.4-3.6' + >>> str(VersionRange(minver='3.4', maxver='4.0')) + '3.4-4.0' + >>> str(VersionRange(maxver='3.7')) + '-3.7' + >>> str(VersionRange(minver='3.5', maxver='3.5')) + '3.5' + >>> str(VersionRange()) + '-' + """ + if self.minver is None is self.maxver: + return '-' + if self.minver == self.maxver: + return str(self.minver) + elif self.minver is None: + return '-{}'.format(self.maxver) + elif self.maxver is None: + return '{}-'.format(self.minver) + else: + return '{}-{}'.format(self.minver, self.maxver) + + def __repr__(self): + """Return version range string. + + >>> repr(VersionRange('5.0-')) + "VersionRange(minver='5.0')" + >>> repr(VersionRange('3.0-3.5')) + "VersionRange(minver='3.0', maxver='3.5')" + """ + result = 'VersionRange(' + if self.minver is not None: + result += "minver='{}'".format(self.minver) + if self.maxver is not None: + result += ", maxver='{}'".format(self.maxver) + result = result.replace('(, ', '(') + return result + ")" + + @staticmethod + def parse(value): + """Return minimum and maximum Python version from given range. + + >>> VersionRange.parse('3.0-') + (Version('3.0'), None) + >>> VersionRange.parse('3.1-3.13') + (Version('3.1'), Version('3.13')) + >>> VersionRange.parse('3.2-4.0') + (Version('3.2'), Version('4.0')) + >>> VersionRange.parse('-3.7') + (None, Version('3.7')) + >>> VersionRange.parse('3.2') + (Version('3.2'), Version('3.2')) + >>> VersionRange.parse('') == VersionRange.parse('-') + True + >>> VersionRange.parse('>= 4.0') + (Version('4.0'), None) + """ + if value in ('', '-'): + return None, None + + match = RANGE_RE.match(value) + if not match: + try: + minv, maxv = VersionRange._parse_pycentral(value) + except Exception: + raise ValueError("version range is invalid: %s" % value) + else: + groups = match.groups() + + if list(groups).count(None) == 3: # only one version is allowed + minv = Version(groups[1]) + return minv, minv + + minv = maxv = None + if groups[0]: # maximum version only + maxv = groups[1] + else: + minv = groups[1] + maxv = groups[3] + + minv = Version(minv) if minv else None + maxv = Version(maxv) if maxv else None + + if maxv and minv and minv > maxv: + raise ValueError("version range is invalid: %s" % value) + + return minv, maxv + + @staticmethod + def _parse_pycentral(value): + """Parse X-Python3-Version. + + >>> VersionRange._parse_pycentral('>= 3.10') + (Version('3.10'), None) + >>> VersionRange._parse_pycentral('<< 4.0') + (None, Version('4.0')) + >>> VersionRange._parse_pycentral('3.1') + (Version('3.1'), Version('3.1')) + >>> VersionRange._parse_pycentral('3.1, 3.2') + (Version('3.1'), None) + """ + + minv = maxv = None + hardcoded = set() + + for item in value.split(','): + item = item.strip() + + match = re.match('>=\s*([\d\.]+)', item) + if match: + minv = match.group(1) + continue + match = re.match('<<\s*([\d\.]+)', item) + if match: + maxv = match.group(1) + continue + match = re.match('^[\d\.]+$', item) + if match: + hardcoded.add(match.group(0)) + + if len(hardcoded) == 1: + ver = hardcoded.pop() + return Version(ver), Version(ver) + + if not minv and hardcoded: + # yeah, no maxv! + minv = sorted(hardcoded)[0] + + return Version(minv) if minv else None, Version(maxv) if maxv else None + + +def default(impl): + """Return default interpreter version for given implementation.""" + if impl not in _defaults.DEFAULT: + raise ValueError("interpreter implementation not supported: %r" % impl) + ver = _defaults.DEFAULT[impl] + return Version(major=ver[0], minor=ver[1]) + + +def supported(impl): + """Return list of supported interpreter versions for given implementation.""" + if impl not in _defaults.SUPPORTED: + raise ValueError("interpreter implementation not supported: %r" % impl) + versions = _defaults.SUPPORTED[impl] + return [Version(major=v[0], minor=v[1]) for v in versions] + + +def get_requested_versions(impl, vrange=None, available=None): + """Return a set of requested and supported Python versions. + + :param impl: interpreter implementation + :param available: if set to `True`, return installed versions only, + if set to `False`, return requested versions that are not installed. + By default returns all requested versions. + :type available: bool + + >>> sorted(get_requested_versions('cpython3', '')) == sorted(supported('cpython3')) + True + >>> sorted(get_requested_versions('cpython3', '-')) == sorted(supported('cpython3')) + True + >>> get_requested_versions('cpython3', '>= 5.0') + set() + """ + if isinstance(vrange, str): + vrange = VersionRange(vrange) + + if not vrange: + versions = set(supported(impl)) + else: + minv = Version(major=0, minor=0) if vrange.minver is None else vrange.minver + maxv = Version(major=99, minor=99) if vrange.maxver is None else vrange.maxver + if minv == maxv: + versions = set([minv] if minv in supported(impl) else tuple()) + else: + versions = set(v for v in supported(impl) if minv <= v < maxv) + + if available is not None: + # to avoid circular imports + global Interpreter + if Interpreter is None: + from dhpython.interpreter import Interpreter + if available: + interpreter = Interpreter(impl=impl) + versions = set(v for v in versions + if exists(interpreter.binary(v))) + elif available is False: + interpreter = Interpreter(impl=impl) + versions = set(v for v in versions + if not exists(interpreter.binary(v))) + + return versions + + +def build_sorted(versions, impl='cpython3'): + """Return sorted list of versions in a build friendly order. + + i.e. default version, if among versions, is sorted last. + + >>> build_sorted([(2, 6), (3, 4), default('cpython3'), (3, 6), (2, 7)])[-1] == default('cpython3') + True + >>> build_sorted(('3.2', (3, 0), '3.1')) + [Version('3.0'), Version('3.1'), Version('3.2')] + """ + default_ver = default(impl) + + result = sorted(Version(v) for v in versions) + try: + result.remove(default_ver) + except ValueError: + pass + else: + result.append(default_ver) + return result |