From 1312af87e5908ac252c0659a60216e96ec5b23bd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 14 Jun 2020 11:11:24 +0200 Subject: Merging upstream version 2.5.1. Signed-off-by: Daniel Baumann --- pre_commit/languages/all.py | 4 +- pre_commit/languages/helpers.py | 4 +- pre_commit/languages/node.py | 2 +- pre_commit/languages/python.py | 147 ++++++++++++++++++++---------------- pre_commit/languages/python_venv.py | 46 ----------- pre_commit/languages/ruby.py | 2 + 6 files changed, 88 insertions(+), 117 deletions(-) delete mode 100644 pre_commit/languages/python_venv.py (limited to 'pre_commit/languages') diff --git a/pre_commit/languages/all.py b/pre_commit/languages/all.py index 8f4ffa8..5609631 100644 --- a/pre_commit/languages/all.py +++ b/pre_commit/languages/all.py @@ -14,7 +14,6 @@ from pre_commit.languages import node from pre_commit.languages import perl from pre_commit.languages import pygrep from pre_commit.languages import python -from pre_commit.languages import python_venv from pre_commit.languages import ruby from pre_commit.languages import rust from pre_commit.languages import script @@ -49,7 +48,6 @@ languages = { 'perl': Language(name='perl', ENVIRONMENT_DIR=perl.ENVIRONMENT_DIR, get_default_version=perl.get_default_version, healthy=perl.healthy, install_environment=perl.install_environment, run_hook=perl.run_hook), # noqa: E501 'pygrep': Language(name='pygrep', ENVIRONMENT_DIR=pygrep.ENVIRONMENT_DIR, get_default_version=pygrep.get_default_version, healthy=pygrep.healthy, install_environment=pygrep.install_environment, run_hook=pygrep.run_hook), # noqa: E501 'python': Language(name='python', ENVIRONMENT_DIR=python.ENVIRONMENT_DIR, get_default_version=python.get_default_version, healthy=python.healthy, install_environment=python.install_environment, run_hook=python.run_hook), # noqa: E501 - 'python_venv': Language(name='python_venv', ENVIRONMENT_DIR=python_venv.ENVIRONMENT_DIR, get_default_version=python_venv.get_default_version, healthy=python_venv.healthy, install_environment=python_venv.install_environment, run_hook=python_venv.run_hook), # noqa: E501 'ruby': Language(name='ruby', ENVIRONMENT_DIR=ruby.ENVIRONMENT_DIR, get_default_version=ruby.get_default_version, healthy=ruby.healthy, install_environment=ruby.install_environment, run_hook=ruby.run_hook), # noqa: E501 'rust': Language(name='rust', ENVIRONMENT_DIR=rust.ENVIRONMENT_DIR, get_default_version=rust.get_default_version, healthy=rust.healthy, install_environment=rust.install_environment, run_hook=rust.run_hook), # noqa: E501 'script': Language(name='script', ENVIRONMENT_DIR=script.ENVIRONMENT_DIR, get_default_version=script.get_default_version, healthy=script.healthy, install_environment=script.install_environment, run_hook=script.run_hook), # noqa: E501 @@ -57,4 +55,6 @@ languages = { 'system': Language(name='system', ENVIRONMENT_DIR=system.ENVIRONMENT_DIR, get_default_version=system.get_default_version, healthy=system.healthy, install_environment=system.install_environment, run_hook=system.run_hook), # noqa: E501 # END GENERATED } +# TODO: fully deprecate `python_venv` +languages['python_venv'] = languages['python'] all_languages = sorted(languages) diff --git a/pre_commit/languages/helpers.py b/pre_commit/languages/helpers.py index b5c95e5..01c65ab 100644 --- a/pre_commit/languages/helpers.py +++ b/pre_commit/languages/helpers.py @@ -18,7 +18,7 @@ from pre_commit.xargs import xargs if TYPE_CHECKING: from typing import NoReturn -FIXED_RANDOM_SEED = 1542676186 +FIXED_RANDOM_SEED = 1542676187 def run_setup_cmd(prefix: Prefix, cmd: Tuple[str, ...]) -> None: @@ -92,7 +92,7 @@ def _shuffled(seq: Sequence[str]) -> List[str]: fixed_random.seed(FIXED_RANDOM_SEED, version=1) seq = list(seq) - random.shuffle(seq, random=fixed_random.random) + fixed_random.shuffle(seq) return seq diff --git a/pre_commit/languages/node.py b/pre_commit/languages/node.py index 9b636d3..26f4919 100644 --- a/pre_commit/languages/node.py +++ b/pre_commit/languages/node.py @@ -79,7 +79,7 @@ def install_environment( # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx?f=255&MSPPError=-2147217396#maxpath if sys.platform == 'win32': # pragma: no cover - envdir = f'\\\\?\\{os.path.normpath(envdir)}' + envdir = fr'\\?\{os.path.normpath(envdir)}' with clean_path_on_failure(envdir): cmd = [ sys.executable, '-mnodeenv', '--prebuilt', '--clean-src', envdir, diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py index 85d8281..6f7c900 100644 --- a/pre_commit/languages/python.py +++ b/pre_commit/languages/python.py @@ -2,8 +2,7 @@ import contextlib import functools import os import sys -from typing import Callable -from typing import ContextManager +from typing import Dict from typing import Generator from typing import Optional from typing import Sequence @@ -26,6 +25,28 @@ from pre_commit.util import cmd_output_b ENVIRONMENT_DIR = 'py_env' +@functools.lru_cache(maxsize=None) +def _version_info(exe: str) -> str: + prog = 'import sys;print(".".join(str(p) for p in sys.version_info))' + try: + return cmd_output(exe, '-S', '-c', prog)[1].strip() + except CalledProcessError: + return f'<>' + + +def _read_pyvenv_cfg(filename: str) -> Dict[str, str]: + ret = {} + with open(filename) as f: + for line in f: + try: + k, v = line.split('=') + except ValueError: # blank line / comment / etc. + continue + else: + ret[k.strip()] = v.strip() + return ret + + def bin_dir(venv: str) -> str: """On windows there's a different directory for the virtualenv""" bin_part = 'Scripts' if os.name == 'nt' else 'bin' @@ -34,6 +55,7 @@ def bin_dir(venv: str) -> str: def get_env_patch(venv: str) -> PatchesT: return ( + ('PIP_DISABLE_PIP_VERSION_CHECK', '1'), ('PYTHONHOME', UNSET), ('VIRTUAL_ENV', venv), ('PATH', (bin_dir(venv), os.pathsep, Var('PATH'))), @@ -45,9 +67,10 @@ def _find_by_py_launcher( ) -> Optional[str]: # pragma: no cover (windows only) if version.startswith('python'): num = version[len('python'):] + cmd = ('py', f'-{num}', '-c', 'import sys; print(sys.executable)') + env = dict(os.environ, PYTHONIOENCODING='UTF-8') try: - cmd = ('py', f'-{num}', '-c', 'import sys; print(sys.executable)') - return cmd_output(*cmd)[1].strip() + return cmd_output(*cmd, env=env)[1].strip() except CalledProcessError: pass return None @@ -115,6 +138,9 @@ def _sys_executable_matches(version: str) -> bool: def norm_version(version: str) -> str: + if version == C.DEFAULT: + return os.path.realpath(sys.executable) + # first see if our current executable is appropriate if _sys_executable_matches(version): return sys.executable @@ -139,70 +165,59 @@ def norm_version(version: str) -> str: return os.path.expanduser(version) -def py_interface( - _dir: str, - _make_venv: Callable[[str, str], None], -) -> Tuple[ - Callable[[Prefix, str], ContextManager[None]], - Callable[[Prefix, str], bool], - Callable[[Hook, Sequence[str], bool], Tuple[int, bytes]], - Callable[[Prefix, str, Sequence[str]], None], -]: - @contextlib.contextmanager - def in_env( - prefix: Prefix, - language_version: str, - ) -> Generator[None, None, None]: - envdir = prefix.path(helpers.environment_dir(_dir, language_version)) - with envcontext(get_env_patch(envdir)): - yield - - def healthy(prefix: Prefix, language_version: str) -> bool: - envdir = helpers.environment_dir(_dir, language_version) - exe_name = 'python.exe' if sys.platform == 'win32' else 'python' - py_exe = prefix.path(bin_dir(envdir), exe_name) - with in_env(prefix, language_version): - retcode, _, _ = cmd_output_b( - py_exe, '-c', 'import ctypes, datetime, io, os, ssl, weakref', - cwd='/', - retcode=None, - ) - return retcode == 0 - - def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, - ) -> Tuple[int, bytes]: - with in_env(hook.prefix, hook.language_version): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) - - def install_environment( - prefix: Prefix, - version: str, - additional_dependencies: Sequence[str], - ) -> None: - directory = helpers.environment_dir(_dir, version) - install = ('python', '-mpip', 'install', '.', *additional_dependencies) - - env_dir = prefix.path(directory) - with clean_path_on_failure(env_dir): - if version != C.DEFAULT: - python = norm_version(version) - else: - python = os.path.realpath(sys.executable) - _make_venv(env_dir, python) - with in_env(prefix, version): - helpers.run_setup_cmd(prefix, install) +@contextlib.contextmanager +def in_env( + prefix: Prefix, + language_version: str, +) -> Generator[None, None, None]: + directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version) + envdir = prefix.path(directory) + with envcontext(get_env_patch(envdir)): + yield - return in_env, healthy, run_hook, install_environment +def healthy(prefix: Prefix, language_version: str) -> bool: + directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version) + envdir = prefix.path(directory) + pyvenv_cfg = os.path.join(envdir, 'pyvenv.cfg') -def make_venv(envdir: str, python: str) -> None: - env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1') - cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python) - cmd_output_b(*cmd, env=env, cwd='/') + # created with "old" virtualenv + if not os.path.exists(pyvenv_cfg): + return False + + exe_name = 'python.exe' if sys.platform == 'win32' else 'python' + py_exe = prefix.path(bin_dir(envdir), exe_name) + cfg = _read_pyvenv_cfg(pyvenv_cfg) + + return ( + 'version_info' in cfg and + _version_info(py_exe) == cfg['version_info'] and ( + 'base-executable' not in cfg or + _version_info(cfg['base-executable']) == cfg['version_info'] + ) + ) -_interface = py_interface(ENVIRONMENT_DIR, make_venv) -in_env, healthy, run_hook, install_environment = _interface +def install_environment( + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], +) -> None: + envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version)) + python = norm_version(version) + venv_cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python) + install_cmd = ('python', '-mpip', 'install', '.', *additional_dependencies) + + with clean_path_on_failure(envdir): + cmd_output_b(*venv_cmd, cwd='/') + with in_env(prefix, version): + helpers.run_setup_cmd(prefix, install_cmd) + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: + with in_env(hook.prefix, hook.language_version): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/python_venv.py b/pre_commit/languages/python_venv.py deleted file mode 100644 index 5404c8b..0000000 --- a/pre_commit/languages/python_venv.py +++ /dev/null @@ -1,46 +0,0 @@ -import os.path - -from pre_commit.languages import python -from pre_commit.util import CalledProcessError -from pre_commit.util import cmd_output -from pre_commit.util import cmd_output_b - -ENVIRONMENT_DIR = 'py_venv' -get_default_version = python.get_default_version - - -def orig_py_exe(exe: str) -> str: # pragma: no cover (platform specific) - """A -mvenv virtualenv made from a -mvirtualenv virtualenv installs - packages to the incorrect location. Attempt to find the _original_ exe - and invoke `-mvenv` from there. - - See: - - https://github.com/pre-commit/pre-commit/issues/755 - - https://github.com/pypa/virtualenv/issues/1095 - - https://bugs.python.org/issue30811 - """ - try: - prefix_script = 'import sys; print(sys.real_prefix)' - _, prefix, _ = cmd_output(exe, '-c', prefix_script) - prefix = prefix.strip() - except CalledProcessError: - # not created from -mvirtualenv - return exe - - if os.name == 'nt': - expected = os.path.join(prefix, 'python.exe') - else: - expected = os.path.join(prefix, 'bin', os.path.basename(exe)) - - if os.path.exists(expected): - return expected - else: - return exe - - -def make_venv(envdir: str, python: str) -> None: - cmd_output_b(orig_py_exe(python), '-mvenv', envdir, cwd='/') - - -_interface = python.py_interface(ENVIRONMENT_DIR, make_venv) -in_env, healthy, run_hook, install_environment = _interface diff --git a/pre_commit/languages/ruby.py b/pre_commit/languages/ruby.py index 61241f8..fe524ec 100644 --- a/pre_commit/languages/ruby.py +++ b/pre_commit/languages/ruby.py @@ -9,6 +9,7 @@ from typing import Tuple import pre_commit.constants as C from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT +from pre_commit.envcontext import UNSET from pre_commit.envcontext import Var from pre_commit.hook import Hook from pre_commit.languages import helpers @@ -28,6 +29,7 @@ def get_env_patch( ) -> PatchesT: # pragma: win32 no cover patches: PatchesT = ( ('GEM_HOME', os.path.join(venv, 'gems')), + ('GEM_PATH', UNSET), ('RBENV_ROOT', venv), ('BUNDLE_IGNORE_CONFIG', '1'), ( -- cgit v1.2.3