From e3829cc5d067599ba8416821f67a936b8b3dd368 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Tue, 3 Nov 2020 07:17:38 +0100 Subject: Merging upstream version 2.8.2. Signed-off-by: Daniel Baumann --- pre_commit/languages/all.py | 4 ++ pre_commit/languages/conda.py | 2 +- pre_commit/languages/coursier.py | 71 +++++++++++++++++++++++++++++++ pre_commit/languages/docker.py | 5 +-- pre_commit/languages/dotnet.py | 90 ++++++++++++++++++++++++++++++++++++++++ pre_commit/languages/helpers.py | 27 ++++++++++++ pre_commit/languages/node.py | 31 ++++++++++---- pre_commit/languages/pygrep.py | 48 +++++++++++++++++++-- pre_commit/languages/python.py | 27 ++++-------- pre_commit/languages/ruby.py | 10 ++--- 10 files changed, 276 insertions(+), 39 deletions(-) create mode 100644 pre_commit/languages/coursier.py create mode 100644 pre_commit/languages/dotnet.py (limited to 'pre_commit/languages') diff --git a/pre_commit/languages/all.py b/pre_commit/languages/all.py index 5609631..9c2e59d 100644 --- a/pre_commit/languages/all.py +++ b/pre_commit/languages/all.py @@ -6,8 +6,10 @@ from typing import Tuple from pre_commit.hook import Hook from pre_commit.languages import conda +from pre_commit.languages import coursier from pre_commit.languages import docker from pre_commit.languages import docker_image +from pre_commit.languages import dotnet from pre_commit.languages import fail from pre_commit.languages import golang from pre_commit.languages import node @@ -40,8 +42,10 @@ class Language(NamedTuple): languages = { # BEGIN GENERATED (testing/gen-languages-all) 'conda': Language(name='conda', ENVIRONMENT_DIR=conda.ENVIRONMENT_DIR, get_default_version=conda.get_default_version, healthy=conda.healthy, install_environment=conda.install_environment, run_hook=conda.run_hook), # noqa: E501 + 'coursier': Language(name='coursier', ENVIRONMENT_DIR=coursier.ENVIRONMENT_DIR, get_default_version=coursier.get_default_version, healthy=coursier.healthy, install_environment=coursier.install_environment, run_hook=coursier.run_hook), # noqa: E501 'docker': Language(name='docker', ENVIRONMENT_DIR=docker.ENVIRONMENT_DIR, get_default_version=docker.get_default_version, healthy=docker.healthy, install_environment=docker.install_environment, run_hook=docker.run_hook), # noqa: E501 'docker_image': Language(name='docker_image', ENVIRONMENT_DIR=docker_image.ENVIRONMENT_DIR, get_default_version=docker_image.get_default_version, healthy=docker_image.healthy, install_environment=docker_image.install_environment, run_hook=docker_image.run_hook), # noqa: E501 + 'dotnet': Language(name='dotnet', ENVIRONMENT_DIR=dotnet.ENVIRONMENT_DIR, get_default_version=dotnet.get_default_version, healthy=dotnet.healthy, install_environment=dotnet.install_environment, run_hook=dotnet.run_hook), # noqa: E501 'fail': Language(name='fail', ENVIRONMENT_DIR=fail.ENVIRONMENT_DIR, get_default_version=fail.get_default_version, healthy=fail.healthy, install_environment=fail.install_environment, run_hook=fail.run_hook), # noqa: E501 'golang': Language(name='golang', ENVIRONMENT_DIR=golang.ENVIRONMENT_DIR, get_default_version=golang.get_default_version, healthy=golang.healthy, install_environment=golang.install_environment, run_hook=golang.run_hook), # noqa: E501 'node': Language(name='node', ENVIRONMENT_DIR=node.ENVIRONMENT_DIR, get_default_version=node.get_default_version, healthy=node.healthy, install_environment=node.install_environment, run_hook=node.run_hook), # noqa: E501 diff --git a/pre_commit/languages/conda.py b/pre_commit/languages/conda.py index 071757a..d634e49 100644 --- a/pre_commit/languages/conda.py +++ b/pre_commit/languages/conda.py @@ -77,7 +77,7 @@ def run_hook( color: bool, ) -> Tuple[int, bytes]: # TODO: Some rare commands need to be run using `conda run` but mostly we - # can run them withot which is much quicker and produces a better + # can run them without which is much quicker and produces a better # output. # cmd = ('conda', 'run', '-p', env_dir) + hook.cmd with in_env(hook.prefix, hook.language_version): diff --git a/pre_commit/languages/coursier.py b/pre_commit/languages/coursier.py new file mode 100644 index 0000000..2841467 --- /dev/null +++ b/pre_commit/languages/coursier.py @@ -0,0 +1,71 @@ +import contextlib +import os +from typing import Generator +from typing import Sequence +from typing import Tuple + +from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT +from pre_commit.envcontext import Var +from pre_commit.hook import Hook +from pre_commit.languages import helpers +from pre_commit.prefix import Prefix +from pre_commit.util import clean_path_on_failure + +ENVIRONMENT_DIR = 'coursier' + +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy + + +def install_environment( + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], +) -> None: # pragma: win32 no cover + helpers.assert_version_default('coursier', version) + helpers.assert_no_additional_deps('coursier', additional_dependencies) + + envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version)) + channel = prefix.path('.pre-commit-channel') + with clean_path_on_failure(envdir): + for app_descriptor in os.listdir(channel): + _, app_file = os.path.split(app_descriptor) + app, _ = os.path.splitext(app_file) + helpers.run_setup_cmd( + prefix, + ( + 'cs', + 'install', + '--default-channels=false', + f'--channel={channel}', + app, + f'--dir={envdir}', + ), + ) + + +def get_env_patch(target_dir: str) -> PatchesT: # pragma: win32 no cover + return ( + ('PATH', (target_dir, os.pathsep, Var('PATH'))), + ) + + +@contextlib.contextmanager +def in_env( + prefix: Prefix, +) -> Generator[None, None, None]: # pragma: win32 no cover + target_dir = prefix.path( + helpers.environment_dir(ENVIRONMENT_DIR, get_default_version()), + ) + with envcontext(get_env_patch(target_dir)): + yield + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: # pragma: win32 no cover + with in_env(hook.prefix): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/docker.py b/pre_commit/languages/docker.py index 9c13119..9d30568 100644 --- a/pre_commit/languages/docker.py +++ b/pre_commit/languages/docker.py @@ -87,9 +87,8 @@ def run_hook( # automated cleanup of docker images. build_docker_image(hook.prefix, pull=False) - hook_cmd = hook.cmd - entry_exe, cmd_rest = hook.cmd[0], hook_cmd[1:] + entry_exe, *cmd_rest = hook.cmd entry_tag = ('--entrypoint', entry_exe, docker_tag(hook.prefix)) - cmd = docker_cmd() + entry_tag + cmd_rest + cmd = (*docker_cmd(), *entry_tag, *cmd_rest) return helpers.run_xargs(hook, cmd, file_args, color=color) diff --git a/pre_commit/languages/dotnet.py b/pre_commit/languages/dotnet.py new file mode 100644 index 0000000..a8abc86 --- /dev/null +++ b/pre_commit/languages/dotnet.py @@ -0,0 +1,90 @@ +import contextlib +import os.path +from typing import Generator +from typing import Sequence +from typing import Tuple + +import pre_commit.constants as C +from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT +from pre_commit.envcontext import Var +from pre_commit.hook import Hook +from pre_commit.languages import helpers +from pre_commit.prefix import Prefix +from pre_commit.util import clean_path_on_failure +from pre_commit.util import rmtree + +ENVIRONMENT_DIR = 'dotnetenv' +BIN_DIR = 'bin' + +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy + + +def get_env_patch(venv: str) -> PatchesT: + return ( + ('PATH', (os.path.join(venv, BIN_DIR), os.pathsep, Var('PATH'))), + ) + + +@contextlib.contextmanager +def in_env(prefix: Prefix) -> Generator[None, None, None]: + directory = helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT) + envdir = prefix.path(directory) + with envcontext(get_env_patch(envdir)): + yield + + +def install_environment( + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], +) -> None: + helpers.assert_version_default('dotnet', version) + helpers.assert_no_additional_deps('dotnet', additional_dependencies) + + envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version)) + with clean_path_on_failure(envdir): + build_dir = 'pre-commit-build' + + # Build & pack nupkg file + helpers.run_setup_cmd( + prefix, + ( + 'dotnet', 'pack', + '--configuration', 'Release', + '--output', build_dir, + ), + ) + + # Determine tool from the packaged file ..nupkg + build_outputs = os.listdir(os.path.join(prefix.prefix_dir, build_dir)) + if len(build_outputs) != 1: + raise NotImplementedError( + f"Can't handle multiple build outputs. Got {build_outputs}", + ) + tool_name = build_outputs[0].split('.')[0] + + # Install to bin dir + helpers.run_setup_cmd( + prefix, + ( + 'dotnet', 'tool', 'install', + '--tool-path', os.path.join(envdir, BIN_DIR), + '--add-source', build_dir, + tool_name, + ), + ) + + # Cleanup build output + for d in ('bin', 'obj', build_dir): + rmtree(prefix.path(d)) + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: + with in_env(hook.prefix): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/helpers.py b/pre_commit/languages/helpers.py index 01c65ab..29138fd 100644 --- a/pre_commit/languages/helpers.py +++ b/pre_commit/languages/helpers.py @@ -1,6 +1,7 @@ import multiprocessing import os import random +import re from typing import Any from typing import List from typing import Optional @@ -10,6 +11,7 @@ from typing import Tuple from typing import TYPE_CHECKING import pre_commit.constants as C +from pre_commit import parse_shebang from pre_commit.hook import Hook from pre_commit.prefix import Prefix from pre_commit.util import cmd_output_b @@ -20,6 +22,31 @@ if TYPE_CHECKING: FIXED_RANDOM_SEED = 1542676187 +SHIMS_RE = re.compile(r'[/\\]shims[/\\]') + + +def exe_exists(exe: str) -> bool: + found = parse_shebang.find_executable(exe) + if found is None: # exe exists + return False + + homedir = os.path.expanduser('~') + try: + common: Optional[str] = os.path.commonpath((found, homedir)) + except ValueError: # on windows, different drives raises ValueError + common = None + + return ( + # it is not in a /shims/ directory + not SHIMS_RE.search(found) and + ( + # the homedir is / (docker, service user, etc.) + os.path.dirname(homedir) == homedir or + # the exe is not contained in the home directory + common != homedir + ) + ) + def run_setup_cmd(prefix: Prefix, cmd: Tuple[str, ...]) -> None: cmd_output_b(*cmd, cwd=prefix.prefix_dir) diff --git a/pre_commit/languages/node.py b/pre_commit/languages/node.py index d99e6f2..8dc4e8b 100644 --- a/pre_commit/languages/node.py +++ b/pre_commit/languages/node.py @@ -7,7 +7,6 @@ from typing import Sequence from typing import Tuple import pre_commit.constants as C -from pre_commit import parse_shebang from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import UNSET @@ -19,9 +18,9 @@ from pre_commit.prefix import Prefix from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output from pre_commit.util import cmd_output_b +from pre_commit.util import rmtree ENVIRONMENT_DIR = 'node_env' -healthy = helpers.basic_healthy @functools.lru_cache(maxsize=1) @@ -31,7 +30,7 @@ def get_default_version() -> str: return C.DEFAULT # if node is already installed, we can save a bunch of setup time by # using the installed version - elif all(parse_shebang.find_executable(exe) for exe in ('node', 'npm')): + elif all(helpers.exe_exists(exe) for exe in ('node', 'npm')): return 'system' else: return C.DEFAULT @@ -73,6 +72,12 @@ def in_env( yield +def healthy(prefix: Prefix, language_version: str) -> bool: + with in_env(prefix, language_version): + retcode, _, _ = cmd_output_b('node', '--version', retcode=None) + return retcode == 0 + + def install_environment( prefix: Prefix, version: str, additional_dependencies: Sequence[str], ) -> None: @@ -94,11 +99,23 @@ def install_environment( with in_env(prefix, version): # https://npm.community/t/npm-install-g-git-vs-git-clone-cd-npm-install-g/5449 # install as if we installed from git - helpers.run_setup_cmd(prefix, ('npm', 'install')) - helpers.run_setup_cmd( - prefix, - ('npm', 'install', '-g', '.', *additional_dependencies), + + local_install_cmd = ( + 'npm', 'install', '--dev', '--prod', + '--ignore-prepublish', '--no-progress', '--no-save', ) + helpers.run_setup_cmd(prefix, local_install_cmd) + + _, pkg, _ = cmd_output('npm', 'pack', cwd=prefix.prefix_dir) + pkg = prefix.path(pkg.strip()) + + install = ('npm', 'install', '-g', pkg, *additional_dependencies) + helpers.run_setup_cmd(prefix, install) + + # clean these up after installation + if prefix.exists('node_modules'): # pragma: win32 no cover + rmtree(prefix.path('node_modules')) + os.remove(pkg) def run_hook( diff --git a/pre_commit/languages/pygrep.py b/pre_commit/languages/pygrep.py index 40adba0..c80d679 100644 --- a/pre_commit/languages/pygrep.py +++ b/pre_commit/languages/pygrep.py @@ -1,6 +1,7 @@ import argparse import re import sys +from typing import NamedTuple from typing import Optional from typing import Pattern from typing import Sequence @@ -45,6 +46,46 @@ def _process_filename_at_once(pattern: Pattern[bytes], filename: str) -> int: return retv +def _process_filename_by_line_negated( + pattern: Pattern[bytes], + filename: str, +) -> int: + with open(filename, 'rb') as f: + for line in f: + if pattern.search(line): + return 0 + else: + output.write_line(filename) + return 1 + + +def _process_filename_at_once_negated( + pattern: Pattern[bytes], + filename: str, +) -> int: + with open(filename, 'rb') as f: + contents = f.read() + match = pattern.search(contents) + if match: + return 0 + else: + output.write_line(filename) + return 1 + + +class Choice(NamedTuple): + multiline: bool + negate: bool + + +FNS = { + Choice(multiline=True, negate=True): _process_filename_at_once_negated, + Choice(multiline=True, negate=False): _process_filename_at_once, + Choice(multiline=False, negate=True): _process_filename_by_line_negated, + Choice(multiline=False, negate=False): _process_filename_by_line, +} + + def run_hook( hook: Hook, file_args: Sequence[str], @@ -64,6 +105,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: ) parser.add_argument('-i', '--ignore-case', action='store_true') parser.add_argument('--multiline', action='store_true') + parser.add_argument('--negate', action='store_true') parser.add_argument('pattern', help='python regex pattern.') parser.add_argument('filenames', nargs='*') args = parser.parse_args(argv) @@ -75,11 +117,9 @@ def main(argv: Optional[Sequence[str]] = None) -> int: pattern = re.compile(args.pattern.encode(), flags) retv = 0 + process_fn = FNS[Choice(multiline=args.multiline, negate=args.negate)] for filename in args.filenames: - if args.multiline: - retv |= _process_filename_at_once(pattern, filename) - else: - retv |= _process_filename_by_line(pattern, filename) + retv |= process_fn(pattern, filename) return retv diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py index 7a68580..65f521c 100644 --- a/pre_commit/languages/python.py +++ b/pre_commit/languages/python.py @@ -114,11 +114,6 @@ def get_default_version() -> str: # pragma: no cover (platform dependent) if _find_by_py_launcher(exe): return exe - # Give a best-effort try for windows - default_folder_name = exe.replace('.', '') - if os.path.exists(fr'C:\{default_folder_name}\python.exe'): - return exe - # We tried! return C.DEFAULT @@ -137,13 +132,11 @@ def _sys_executable_matches(version: str) -> bool: return sys.version_info[:len(info)] == info -def norm_version(version: str) -> str: - if version == C.DEFAULT: - return os.path.realpath(sys.executable) - - # first see if our current executable is appropriate - if _sys_executable_matches(version): - return sys.executable +def norm_version(version: str) -> Optional[str]: + if version == C.DEFAULT: # use virtualenv's default + return None + elif _sys_executable_matches(version): # virtualenv defaults to our exe + return None if os.name == 'nt': # pragma: no cover (windows) version_exec = _find_by_py_launcher(version) @@ -155,12 +148,6 @@ def norm_version(version: str) -> str: if version_exec and version_exec != version: return version_exec - # If it is in the form pythonx.x search in the default - # place on windows - if version.startswith('python'): - default_folder_name = version.replace('.', '') - return fr'C:\{default_folder_name}\python.exe' - # Otherwise assume it is a path return os.path.expanduser(version) @@ -205,8 +192,10 @@ def install_environment( additional_dependencies: Sequence[str], ) -> None: envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version)) + venv_cmd = [sys.executable, '-mvirtualenv', envdir] python = norm_version(version) - venv_cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python) + if python is not None: + venv_cmd.extend(('-p', python)) install_cmd = ('python', '-mpip', 'install', '.', *additional_dependencies) with clean_path_on_failure(envdir): diff --git a/pre_commit/languages/ruby.py b/pre_commit/languages/ruby.py index 73b23cc..1a0f0c7 100644 --- a/pre_commit/languages/ruby.py +++ b/pre_commit/languages/ruby.py @@ -8,7 +8,6 @@ from typing import Sequence from typing import Tuple import pre_commit.constants as C -from pre_commit import parse_shebang from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import UNSET @@ -26,7 +25,7 @@ healthy = helpers.basic_healthy @functools.lru_cache(maxsize=1) def get_default_version() -> str: - if all(parse_shebang.find_executable(exe) for exe in ('ruby', 'gem')): + if all(helpers.exe_exists(exe) for exe in ('ruby', 'gem')): return 'system' else: return C.DEFAULT @@ -122,8 +121,8 @@ def install_environment( # Need to call this before installing so rbenv's directories # are set up helpers.run_setup_cmd(prefix, ('rbenv', 'init', '-')) - # XXX: this will *always* fail if `version == C.DEFAULT` - _install_ruby(prefix, version) + if version != C.DEFAULT: + _install_ruby(prefix, version) # Need to call this after installing to set up the shims helpers.run_setup_cmd(prefix, ('rbenv', 'rehash')) @@ -134,7 +133,8 @@ def install_environment( helpers.run_setup_cmd( prefix, ( - 'gem', 'install', '--no-document', + 'gem', 'install', + '--no-document', '--no-format-executable', *prefix.star('.gem'), *additional_dependencies, ), ) -- cgit v1.2.3