diff options
Diffstat (limited to 'pre_commit')
37 files changed, 805 insertions, 920 deletions
diff --git a/pre_commit/clientlib.py b/pre_commit/clientlib.py index da6ca2b..e191d3a 100644 --- a/pre_commit/clientlib.py +++ b/pre_commit/clientlib.py @@ -1,6 +1,5 @@ from __future__ import annotations -import argparse import functools import logging import re @@ -13,14 +12,9 @@ import cfgv from identify.identify import ALL_TAGS import pre_commit.constants as C -from pre_commit.color import add_color_option -from pre_commit.commands.validate_config import validate_config -from pre_commit.commands.validate_manifest import validate_manifest from pre_commit.errors import FatalError from pre_commit.languages.all import all_languages -from pre_commit.logging_handler import logging_handler -from pre_commit.util import parse_version -from pre_commit.util import yaml_load +from pre_commit.yaml import yaml_load logger = logging.getLogger('pre_commit') @@ -35,6 +29,11 @@ def check_type_tag(tag: str) -> None: ) +def parse_version(s: str) -> tuple[int, ...]: + """poor man's version comparison""" + return tuple(int(p) for p in s.split('.')) + + def check_min_version(version: str) -> None: if parse_version(version) > parse_version(C.VERSION): raise cfgv.ValidationError( @@ -44,14 +43,6 @@ def check_min_version(version: str) -> None: ) -def _make_argparser(filenames_help: str) -> argparse.ArgumentParser: - parser = argparse.ArgumentParser() - parser.add_argument('filenames', nargs='*', help=filenames_help) - parser.add_argument('-V', '--version', action='version', version=C.VERSION) - add_color_option(parser) - return parser - - MANIFEST_HOOK_DICT = cfgv.Map( 'Hook', 'id', @@ -97,25 +88,11 @@ load_manifest = functools.partial( ) -def validate_manifest_main(argv: Sequence[str] | None = None) -> int: - parser = _make_argparser('Manifest filenames.') - args = parser.parse_args(argv) - - with logging_handler(args.color): - logger.warning( - 'pre-commit-validate-manifest is deprecated -- ' - 'use `pre-commit validate-manifest` instead.', - ) - - return validate_manifest(args.filenames) - - LOCAL = 'local' META = 'meta' -# should inherit from cfgv.Conditional if sha support is dropped -class WarnMutableRev(cfgv.ConditionalOptional): +class WarnMutableRev(cfgv.Conditional): def check(self, dct: dict[str, Any]) -> None: super().check(dct) @@ -171,36 +148,6 @@ class OptionalSensibleRegexAtTop(cfgv.OptionalNoDefault): ) -class MigrateShaToRev: - key = 'rev' - - @staticmethod - def _cond(key: str) -> cfgv.Conditional: - return cfgv.Conditional( - key, cfgv.check_string, - condition_key='repo', - condition_value=cfgv.NotIn(LOCAL, META), - ensure_absent=True, - ) - - def check(self, dct: dict[str, Any]) -> None: - if dct.get('repo') in {LOCAL, META}: - self._cond('rev').check(dct) - self._cond('sha').check(dct) - elif 'sha' in dct and 'rev' in dct: - raise cfgv.ValidationError('Cannot specify both sha and rev') - elif 'sha' in dct: - self._cond('sha').check(dct) - else: - self._cond('rev').check(dct) - - def apply_default(self, dct: dict[str, Any]) -> None: - if 'sha' in dct: - dct['rev'] = dct.pop('sha') - - remove_default = cfgv.Required.remove_default - - def _entry(modname: str) -> str: """the hook `entry` is passed through `shlex.split()` by the command runner, so to prevent issues with spaces and backslashes (on Windows) @@ -324,14 +271,11 @@ CONFIG_REPO_DICT = cfgv.Map( 'repo', META, ), - MigrateShaToRev(), WarnMutableRev( - 'rev', - cfgv.check_string, - '', - 'repo', - cfgv.NotIn(LOCAL, META), - True, + 'rev', cfgv.check_string, + condition_key='repo', + condition_value=cfgv.NotIn(LOCAL, META), + ensure_absent=True, ), cfgv.WarnAdditionalKeys(('repo', 'rev', 'hooks'), warn_unknown_keys_repo), ) @@ -391,35 +335,9 @@ class InvalidConfigError(FatalError): pass -def ordered_load_normalize_legacy_config(contents: str) -> dict[str, Any]: - data = yaml_load(contents) - if isinstance(data, list): - logger.warning( - 'normalizing pre-commit configuration to a top-level map. ' - 'support for top level list will be removed in a future version. ' - 'run: `pre-commit migrate-config` to automatically fix this.', - ) - return {'repos': data} - else: - return data - - load_config = functools.partial( cfgv.load_from_filename, schema=CONFIG_SCHEMA, - load_strategy=ordered_load_normalize_legacy_config, + load_strategy=yaml_load, exc_tp=InvalidConfigError, ) - - -def validate_config_main(argv: Sequence[str] | None = None) -> int: - parser = _make_argparser('Config filenames.') - args = parser.parse_args(argv) - - with logging_handler(args.color): - logger.warning( - 'pre-commit-validate-config is deprecated -- ' - 'use `pre-commit validate-config` instead.', - ) - - return validate_config(args.filenames) diff --git a/pre_commit/commands/autoupdate.py b/pre_commit/commands/autoupdate.py index d5352e5..7ed6e77 100644 --- a/pre_commit/commands/autoupdate.py +++ b/pre_commit/commands/autoupdate.py @@ -2,6 +2,7 @@ from __future__ import annotations import os.path import re +import tempfile from typing import Any from typing import NamedTuple from typing import Sequence @@ -19,9 +20,8 @@ from pre_commit.store import Store from pre_commit.util import CalledProcessError from pre_commit.util import cmd_output from pre_commit.util import cmd_output_b -from pre_commit.util import tmpdir -from pre_commit.util import yaml_dump -from pre_commit.util import yaml_load +from pre_commit.yaml import yaml_dump +from pre_commit.yaml import yaml_load class RevInfo(NamedTuple): @@ -47,7 +47,7 @@ class RevInfo(NamedTuple): 'FETCH_HEAD', '--tags', '--exact', ) - with tmpdir() as tmp: + with tempfile.TemporaryDirectory() as tmp: git.init_repo(tmp, self.repo) cmd_output_b( *git_cmd, 'fetch', 'origin', 'HEAD', '--tags', diff --git a/pre_commit/commands/migrate_config.py b/pre_commit/commands/migrate_config.py index c3d0a50..6f7af4e 100644 --- a/pre_commit/commands/migrate_config.py +++ b/pre_commit/commands/migrate_config.py @@ -3,9 +3,11 @@ from __future__ import annotations import re import textwrap +import cfgv import yaml -from pre_commit.util import yaml_load +from pre_commit.clientlib import InvalidConfigError +from pre_commit.yaml import yaml_load def _is_header_line(line: str) -> bool: @@ -44,6 +46,13 @@ def migrate_config(config_file: str, quiet: bool = False) -> int: with open(config_file) as f: orig_contents = contents = f.read() + with cfgv.reraise_as(InvalidConfigError): + with cfgv.validate_context(f'File {config_file}'): + try: + yaml_load(orig_contents) + except Exception as e: + raise cfgv.ValidationError(str(e)) + contents = _migrate_map(contents) contents = _migrate_sha_to_rev(contents) diff --git a/pre_commit/commands/run.py b/pre_commit/commands/run.py index 429e04c..e44e703 100644 --- a/pre_commit/commands/run.py +++ b/pre_commit/commands/run.py @@ -189,7 +189,16 @@ def _run_single_hook( filenames = () time_before = time.time() language = languages[hook.language] - retcode, out = language.run_hook(hook, filenames, use_color) + with language.in_env(hook.prefix, hook.language_version): + retcode, out = language.run_hook( + hook.prefix, + hook.entry, + hook.args, + filenames, + is_local=hook.src == 'local', + require_serial=hook.require_serial, + color=use_color, + ) duration = round(time.time() - time_before, 2) or 0 diff_after = _get_diff() diff --git a/pre_commit/commands/try_repo.py b/pre_commit/commands/try_repo.py index ef099f5..539ed3c 100644 --- a/pre_commit/commands/try_repo.py +++ b/pre_commit/commands/try_repo.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse import logging import os.path +import tempfile import pre_commit.constants as C from pre_commit import git @@ -11,9 +12,8 @@ from pre_commit.clientlib import load_manifest from pre_commit.commands.run import run from pre_commit.store import Store from pre_commit.util import cmd_output_b -from pre_commit.util import tmpdir -from pre_commit.util import yaml_dump from pre_commit.xargs import xargs +from pre_commit.yaml import yaml_dump logger = logging.getLogger(__name__) @@ -49,7 +49,7 @@ def _repo_ref(tmpdir: str, repo: str, ref: str | None) -> tuple[str, str]: def try_repo(args: argparse.Namespace) -> int: - with tmpdir() as tempdir: + with tempfile.TemporaryDirectory() as tempdir: repo, ref = _repo_ref(tempdir, args.repo, args.ref) store = Store(tempdir) diff --git a/pre_commit/commands/validate_config.py b/pre_commit/commands/validate_config.py index 91bb017..24bd313 100644 --- a/pre_commit/commands/validate_config.py +++ b/pre_commit/commands/validate_config.py @@ -1,9 +1,11 @@ from __future__ import annotations +from typing import Sequence + from pre_commit import clientlib -def validate_config(filenames: list[str]) -> int: +def validate_config(filenames: Sequence[str]) -> int: ret = 0 for filename in filenames: diff --git a/pre_commit/commands/validate_manifest.py b/pre_commit/commands/validate_manifest.py index 372a638..419031a 100644 --- a/pre_commit/commands/validate_manifest.py +++ b/pre_commit/commands/validate_manifest.py @@ -1,9 +1,11 @@ from __future__ import annotations +from typing import Sequence + from pre_commit import clientlib -def validate_manifest(filenames: list[str]) -> int: +def validate_manifest(filenames: Sequence[str]) -> int: ret = 0 for filename in filenames: diff --git a/pre_commit/constants.py b/pre_commit/constants.py index 5bc4ae9..3f03cee 100644 --- a/pre_commit/constants.py +++ b/pre_commit/constants.py @@ -1,21 +1,14 @@ from __future__ import annotations -import sys - -if sys.version_info >= (3, 8): # pragma: >=3.8 cover - import importlib.metadata as importlib_metadata -else: # pragma: <3.8 cover - import importlib_metadata +import importlib.metadata CONFIG_FILE = '.pre-commit-config.yaml' MANIFEST_FILE = '.pre-commit-hooks.yaml' -# Bump when installation changes in a backwards / forwards incompatible way -INSTALLED_STATE_VERSION = '1' # Bump when modifying `empty_template` LOCAL_REPO_VERSION = '1' -VERSION = importlib_metadata.version('pre_commit') +VERSION = importlib.metadata.version('pre_commit') # `manual` is not invoked by any installed git hook. See #719 STAGES = ( diff --git a/pre_commit/git.py b/pre_commit/git.py index a76118f..333dc7b 100644 --- a/pre_commit/git.py +++ b/pre_commit/git.py @@ -93,11 +93,6 @@ def get_git_common_dir(git_root: str = '.') -> str: return get_git_dir(git_root) -def get_remote_url(git_root: str) -> str: - _, out, _ = cmd_output('git', 'config', 'remote.origin.url', cwd=git_root) - return out.strip() - - def is_in_merge_conflict() -> bool: git_dir = get_git_dir('.') return ( diff --git a/pre_commit/hook.py b/pre_commit/hook.py index 202abb3..6d436ca 100644 --- a/pre_commit/hook.py +++ b/pre_commit/hook.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -import shlex from typing import Any from typing import NamedTuple from typing import Sequence @@ -38,10 +37,6 @@ class Hook(NamedTuple): verbose: bool @property - def cmd(self) -> tuple[str, ...]: - return (*shlex.split(self.entry), *self.args) - - @property def install_key(self) -> tuple[Prefix, str, str, tuple[str, ...]]: return ( self.prefix, diff --git a/pre_commit/languages/all.py b/pre_commit/languages/all.py index cfd42ce..d952ae1 100644 --- a/pre_commit/languages/all.py +++ b/pre_commit/languages/all.py @@ -1,10 +1,9 @@ from __future__ import annotations -from typing import Callable -from typing import NamedTuple +from typing import ContextManager +from typing import Protocol from typing import Sequence -from pre_commit.hook import Hook from pre_commit.languages import conda from pre_commit.languages import coursier from pre_commit.languages import dart @@ -27,44 +26,74 @@ from pre_commit.languages import system from pre_commit.prefix import Prefix -class Language(NamedTuple): - name: str +class Language(Protocol): # Use `None` for no installation / environment - ENVIRONMENT_DIR: str | None + @property + def ENVIRONMENT_DIR(self) -> str | None: ... # return a value to replace `'default` for `language_version` - get_default_version: Callable[[], str] + def get_default_version(self) -> str: ... + # return whether the environment is healthy (or should be rebuilt) - health_check: Callable[[Prefix, str], str | None] + def health_check( + self, + prefix: Prefix, + language_version: str, + ) -> str | None: + ... + # install a repository for the given language and language_version - install_environment: Callable[[Prefix, str, Sequence[str]], None] + def install_environment( + self, + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], + ) -> None: + ... + + # modify the environment for hook execution + def in_env( + self, + prefix: Prefix, + version: str, + ) -> ContextManager[None]: + ... + # execute a hook and return the exit code and output - run_hook: Callable[[Hook, Sequence[str], bool], tuple[int, bytes]] + def run_hook( + self, + prefix: Prefix, + entry: str, + args: Sequence[str], + file_args: Sequence[str], + *, + is_local: bool, + require_serial: bool, + color: bool, + ) -> tuple[int, bytes]: + ... -# TODO: back to modules + Protocol: https://github.com/python/mypy/issues/5018 -languages = { - # BEGIN GENERATED (testing/gen-languages-all) - 'conda': Language(name='conda', ENVIRONMENT_DIR=conda.ENVIRONMENT_DIR, get_default_version=conda.get_default_version, health_check=conda.health_check, install_environment=conda.install_environment, run_hook=conda.run_hook), # noqa: E501 - 'coursier': Language(name='coursier', ENVIRONMENT_DIR=coursier.ENVIRONMENT_DIR, get_default_version=coursier.get_default_version, health_check=coursier.health_check, install_environment=coursier.install_environment, run_hook=coursier.run_hook), # noqa: E501 - 'dart': Language(name='dart', ENVIRONMENT_DIR=dart.ENVIRONMENT_DIR, get_default_version=dart.get_default_version, health_check=dart.health_check, install_environment=dart.install_environment, run_hook=dart.run_hook), # noqa: E501 - 'docker': Language(name='docker', ENVIRONMENT_DIR=docker.ENVIRONMENT_DIR, get_default_version=docker.get_default_version, health_check=docker.health_check, install_environment=docker.install_environment, run_hook=docker.run_hook), # noqa: E501 - 'docker_image': Language(name='docker_image', ENVIRONMENT_DIR=docker_image.ENVIRONMENT_DIR, get_default_version=docker_image.get_default_version, health_check=docker_image.health_check, install_environment=docker_image.install_environment, run_hook=docker_image.run_hook), # noqa: E501 - 'dotnet': Language(name='dotnet', ENVIRONMENT_DIR=dotnet.ENVIRONMENT_DIR, get_default_version=dotnet.get_default_version, health_check=dotnet.health_check, install_environment=dotnet.install_environment, run_hook=dotnet.run_hook), # noqa: E501 - 'fail': Language(name='fail', ENVIRONMENT_DIR=fail.ENVIRONMENT_DIR, get_default_version=fail.get_default_version, health_check=fail.health_check, install_environment=fail.install_environment, run_hook=fail.run_hook), # noqa: E501 - 'golang': Language(name='golang', ENVIRONMENT_DIR=golang.ENVIRONMENT_DIR, get_default_version=golang.get_default_version, health_check=golang.health_check, install_environment=golang.install_environment, run_hook=golang.run_hook), # noqa: E501 - 'lua': Language(name='lua', ENVIRONMENT_DIR=lua.ENVIRONMENT_DIR, get_default_version=lua.get_default_version, health_check=lua.health_check, install_environment=lua.install_environment, run_hook=lua.run_hook), # noqa: E501 - 'node': Language(name='node', ENVIRONMENT_DIR=node.ENVIRONMENT_DIR, get_default_version=node.get_default_version, health_check=node.health_check, install_environment=node.install_environment, run_hook=node.run_hook), # noqa: E501 - 'perl': Language(name='perl', ENVIRONMENT_DIR=perl.ENVIRONMENT_DIR, get_default_version=perl.get_default_version, health_check=perl.health_check, install_environment=perl.install_environment, run_hook=perl.run_hook), # noqa: E501 - 'pygrep': Language(name='pygrep', ENVIRONMENT_DIR=pygrep.ENVIRONMENT_DIR, get_default_version=pygrep.get_default_version, health_check=pygrep.health_check, install_environment=pygrep.install_environment, run_hook=pygrep.run_hook), # noqa: E501 - 'python': Language(name='python', ENVIRONMENT_DIR=python.ENVIRONMENT_DIR, get_default_version=python.get_default_version, health_check=python.health_check, install_environment=python.install_environment, run_hook=python.run_hook), # noqa: E501 - 'r': Language(name='r', ENVIRONMENT_DIR=r.ENVIRONMENT_DIR, get_default_version=r.get_default_version, health_check=r.health_check, install_environment=r.install_environment, run_hook=r.run_hook), # noqa: E501 - 'ruby': Language(name='ruby', ENVIRONMENT_DIR=ruby.ENVIRONMENT_DIR, get_default_version=ruby.get_default_version, health_check=ruby.health_check, install_environment=ruby.install_environment, run_hook=ruby.run_hook), # noqa: E501 - 'rust': Language(name='rust', ENVIRONMENT_DIR=rust.ENVIRONMENT_DIR, get_default_version=rust.get_default_version, health_check=rust.health_check, install_environment=rust.install_environment, run_hook=rust.run_hook), # noqa: E501 - 'script': Language(name='script', ENVIRONMENT_DIR=script.ENVIRONMENT_DIR, get_default_version=script.get_default_version, health_check=script.health_check, install_environment=script.install_environment, run_hook=script.run_hook), # noqa: E501 - 'swift': Language(name='swift', ENVIRONMENT_DIR=swift.ENVIRONMENT_DIR, get_default_version=swift.get_default_version, health_check=swift.health_check, install_environment=swift.install_environment, run_hook=swift.run_hook), # noqa: E501 - 'system': Language(name='system', ENVIRONMENT_DIR=system.ENVIRONMENT_DIR, get_default_version=system.get_default_version, health_check=system.health_check, install_environment=system.install_environment, run_hook=system.run_hook), # noqa: E501 - # END GENERATED +languages: dict[str, Language] = { + 'conda': conda, + 'coursier': coursier, + 'dart': dart, + 'docker': docker, + 'docker_image': docker_image, + 'dotnet': dotnet, + 'fail': fail, + 'golang': golang, + 'lua': lua, + 'node': node, + 'perl': perl, + 'pygrep': pygrep, + 'python': python, + 'r': r, + 'ruby': ruby, + 'rust': rust, + 'script': script, + 'swift': swift, + 'system': system, + # TODO: fully deprecate `python_venv` + 'python_venv': python, } -# TODO: fully deprecate `python_venv` -languages['python_venv'] = languages['python'] all_languages = sorted(languages) diff --git a/pre_commit/languages/conda.py b/pre_commit/languages/conda.py index f0195e4..e2fb019 100644 --- a/pre_commit/languages/conda.py +++ b/pre_commit/languages/conda.py @@ -10,15 +10,14 @@ from pre_commit.envcontext import PatchesT from pre_commit.envcontext import SubstitutionT from pre_commit.envcontext import UNSET from pre_commit.envcontext import Var -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.prefix import Prefix -from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output_b ENVIRONMENT_DIR = 'conda' get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check +run_hook = helpers.basic_run_hook def get_env_patch(env: str) -> PatchesT: @@ -41,12 +40,8 @@ def get_env_patch(env: str) -> PatchesT: @contextlib.contextmanager -def in_env( - prefix: Prefix, - language_version: str, -) -> Generator[None, None, None]: - directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version) - envdir = prefix.path(directory) +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield @@ -66,31 +61,16 @@ def install_environment( additional_dependencies: Sequence[str], ) -> None: helpers.assert_version_default('conda', version) - directory = helpers.environment_dir(ENVIRONMENT_DIR, version) conda_exe = _conda_exe() - env_dir = prefix.path(directory) - with clean_path_on_failure(env_dir): + env_dir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) + cmd_output_b( + conda_exe, 'env', 'create', '-p', env_dir, '--file', + 'environment.yml', cwd=prefix.prefix_dir, + ) + if additional_dependencies: cmd_output_b( - conda_exe, 'env', 'create', '-p', env_dir, '--file', - 'environment.yml', cwd=prefix.prefix_dir, + conda_exe, 'install', '-p', env_dir, *additional_dependencies, + cwd=prefix.prefix_dir, ) - if additional_dependencies: - cmd_output_b( - conda_exe, 'install', '-p', env_dir, *additional_dependencies, - cwd=prefix.prefix_dir, - ) - - -def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, -) -> tuple[int, bytes]: - # TODO: Some rare commands need to be run using `conda run` but mostly we - # can run them without which is much quicker and produces a better - # output. - # cmd = ('conda', 'run', '-p', env_dir) + hook.cmd - with in_env(hook.prefix, hook.language_version): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/coursier.py b/pre_commit/languages/coursier.py index 9fe43eb..6075758 100644 --- a/pre_commit/languages/coursier.py +++ b/pre_commit/languages/coursier.py @@ -1,81 +1,76 @@ from __future__ import annotations import contextlib -import os +import os.path from typing import Generator from typing import Sequence from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var -from pre_commit.hook import Hook +from pre_commit.errors import FatalError from pre_commit.languages import helpers from pre_commit.parse_shebang import find_executable from pre_commit.prefix import Prefix -from pre_commit.util import clean_path_on_failure ENVIRONMENT_DIR = 'coursier' get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check +run_hook = helpers.basic_run_hook def install_environment( prefix: Prefix, version: str, additional_dependencies: Sequence[str], -) -> None: # pragma: win32 no cover +) -> None: helpers.assert_version_default('coursier', version) - helpers.assert_no_additional_deps('coursier', additional_dependencies) # Support both possible executable names (either "cs" or "coursier") - executable = find_executable('cs') or find_executable('coursier') - if executable is None: + cs = find_executable('cs') or find_executable('coursier') + if cs is None: raise AssertionError( 'pre-commit requires system-installed "cs" or "coursier" ' 'executables in the application search path', ) - envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version)) - channel = prefix.path('.pre-commit-channel') - with clean_path_on_failure(envdir): - for app_descriptor in os.listdir(channel): - _, app_file = os.path.split(app_descriptor) - app, _ = os.path.splitext(app_file) - helpers.run_setup_cmd( - prefix, - ( - executable, - 'install', + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) + + def _install(*opts: str) -> None: + assert cs is not None + helpers.run_setup_cmd(prefix, (cs, 'fetch', *opts)) + helpers.run_setup_cmd(prefix, (cs, 'install', '--dir', envdir, *opts)) + + with in_env(prefix, version): + channel = prefix.path('.pre-commit-channel') + if os.path.isdir(channel): + for app_descriptor in os.listdir(channel): + _, app_file = os.path.split(app_descriptor) + app, _ = os.path.splitext(app_file) + _install( '--default-channels=false', - f'--channel={channel}', + '--channel', channel, app, - f'--dir={envdir}', - ), + ) + elif not additional_dependencies: + raise FatalError( + 'expected .pre-commit-channel dir or additional_dependencies', ) + if additional_dependencies: + _install(*additional_dependencies) -def get_env_patch(target_dir: str) -> PatchesT: # pragma: win32 no cover + +def get_env_patch(target_dir: str) -> PatchesT: return ( ('PATH', (target_dir, os.pathsep, Var('PATH'))), + ('COURSIER_CACHE', os.path.join(target_dir, '.cs-cache')), ) @contextlib.contextmanager -def in_env( - prefix: Prefix, -) -> Generator[None, None, None]: # pragma: win32 no cover - target_dir = prefix.path( - helpers.environment_dir(ENVIRONMENT_DIR, get_default_version()), - ) - with envcontext(get_env_patch(target_dir)): +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) + with envcontext(get_env_patch(envdir)): yield - - -def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, -) -> tuple[int, bytes]: # pragma: win32 no cover - with in_env(hook.prefix): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/dart.py b/pre_commit/languages/dart.py index 55ecbf4..e3c1c58 100644 --- a/pre_commit/languages/dart.py +++ b/pre_commit/languages/dart.py @@ -7,21 +7,19 @@ import tempfile from typing import Generator from typing import Sequence -import pre_commit.constants as C from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.prefix import Prefix -from pre_commit.util import clean_path_on_failure from pre_commit.util import win_exe -from pre_commit.util import yaml_load +from pre_commit.yaml import yaml_load ENVIRONMENT_DIR = 'dartenv' get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check +run_hook = helpers.basic_run_hook def get_env_patch(venv: str) -> PatchesT: @@ -31,9 +29,8 @@ def get_env_patch(venv: str) -> PatchesT: @contextlib.contextmanager -def in_env(prefix: Prefix) -> Generator[None, None, None]: - directory = helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT) - envdir = prefix.path(directory) +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield @@ -45,7 +42,7 @@ def install_environment( ) -> None: helpers.assert_version_default('dart', version) - envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version)) + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) bin_dir = os.path.join(envdir, 'bin') def _install_dir(prefix_p: Prefix, pub_cache: str) -> None: @@ -67,44 +64,34 @@ def install_environment( env=dart_env, ) - with clean_path_on_failure(envdir): - os.makedirs(bin_dir) + os.makedirs(bin_dir) - with tempfile.TemporaryDirectory() as tmp: - _install_dir(prefix, tmp) + with tempfile.TemporaryDirectory() as tmp: + _install_dir(prefix, tmp) - for dep_s in additional_dependencies: - with tempfile.TemporaryDirectory() as dep_tmp: - dep, _, version = dep_s.partition(':') - if version: - dep_cmd: tuple[str, ...] = (dep, '--version', version) - else: - dep_cmd = (dep,) + for dep_s in additional_dependencies: + with tempfile.TemporaryDirectory() as dep_tmp: + dep, _, version = dep_s.partition(':') + if version: + dep_cmd: tuple[str, ...] = (dep, '--version', version) + else: + dep_cmd = (dep,) - helpers.run_setup_cmd( - prefix, - ('dart', 'pub', 'cache', 'add', *dep_cmd), - env={**os.environ, 'PUB_CACHE': dep_tmp}, - ) + helpers.run_setup_cmd( + prefix, + ('dart', 'pub', 'cache', 'add', *dep_cmd), + env={**os.environ, 'PUB_CACHE': dep_tmp}, + ) - # try and find the 'pubspec.yaml' that just got added - for root, _, filenames in os.walk(dep_tmp): - if 'pubspec.yaml' in filenames: - with tempfile.TemporaryDirectory() as copied: - pkg = os.path.join(copied, 'pkg') - shutil.copytree(root, pkg) - _install_dir(Prefix(pkg), dep_tmp) - break - else: - raise AssertionError( - f'could not find pubspec.yaml for {dep_s}', - ) - - -def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, -) -> tuple[int, bytes]: - with in_env(hook.prefix): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) + # try and find the 'pubspec.yaml' that just got added + for root, _, filenames in os.walk(dep_tmp): + if 'pubspec.yaml' in filenames: + with tempfile.TemporaryDirectory() as copied: + pkg = os.path.join(copied, 'pkg') + shutil.copytree(root, pkg) + _install_dir(Prefix(pkg), dep_tmp) + break + else: + raise AssertionError( + f'could not find pubspec.yaml for {dep_s}', + ) diff --git a/pre_commit/languages/docker.py b/pre_commit/languages/docker.py index eea9f76..e80c959 100644 --- a/pre_commit/languages/docker.py +++ b/pre_commit/languages/docker.py @@ -5,18 +5,16 @@ import json import os from typing import Sequence -import pre_commit.constants as C -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.prefix import Prefix from pre_commit.util import CalledProcessError -from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output_b ENVIRONMENT_DIR = 'docker' PRE_COMMIT_LABEL = 'PRE_COMMIT' get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check +in_env = helpers.no_env # no special environment for docker def _is_in_docker() -> bool: @@ -95,15 +93,12 @@ def install_environment( helpers.assert_version_default('docker', version) helpers.assert_no_additional_deps('docker', additional_dependencies) - directory = prefix.path( - helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), - ) + directory = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) # Docker doesn't really have relevant disk environment, but pre-commit # still needs to cleanup its state files on failure - with clean_path_on_failure(directory): - build_docker_image(prefix, pull=True) - os.mkdir(directory) + build_docker_image(prefix, pull=True) + os.mkdir(directory) def get_docker_user() -> tuple[str, ...]: # pragma: win32 no cover @@ -127,16 +122,26 @@ def docker_cmd() -> tuple[str, ...]: # pragma: win32 no cover def run_hook( - hook: Hook, + prefix: Prefix, + entry: str, + args: Sequence[str], file_args: Sequence[str], + *, + is_local: bool, + require_serial: bool, color: bool, ) -> tuple[int, bytes]: # pragma: win32 no cover # Rebuild the docker image in case it has gone missing, as many people do # automated cleanup of docker images. - build_docker_image(hook.prefix, pull=False) + build_docker_image(prefix, pull=False) - entry_exe, *cmd_rest = hook.cmd + entry_exe, *cmd_rest = helpers.hook_cmd(entry, args) - entry_tag = ('--entrypoint', entry_exe, docker_tag(hook.prefix)) + entry_tag = ('--entrypoint', entry_exe, docker_tag(prefix)) cmd = (*docker_cmd(), *entry_tag, *cmd_rest) - return helpers.run_xargs(hook, cmd, file_args, color=color) + return helpers.run_xargs( + cmd, + file_args, + require_serial=require_serial, + color=color, + ) diff --git a/pre_commit/languages/docker_image.py b/pre_commit/languages/docker_image.py index daa4d1b..8e5f2c0 100644 --- a/pre_commit/languages/docker_image.py +++ b/pre_commit/languages/docker_image.py @@ -2,20 +2,31 @@ from __future__ import annotations from typing import Sequence -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.languages.docker import docker_cmd +from pre_commit.prefix import Prefix ENVIRONMENT_DIR = None get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check install_environment = helpers.no_install +in_env = helpers.no_env def run_hook( - hook: Hook, + prefix: Prefix, + entry: str, + args: Sequence[str], file_args: Sequence[str], + *, + is_local: bool, + require_serial: bool, color: bool, ) -> tuple[int, bytes]: # pragma: win32 no cover - cmd = docker_cmd() + hook.cmd - return helpers.run_xargs(hook, cmd, file_args, color=color) + cmd = docker_cmd() + helpers.hook_cmd(entry, args) + return helpers.run_xargs( + cmd, + file_args, + require_serial=require_serial, + color=color, + ) diff --git a/pre_commit/languages/dotnet.py b/pre_commit/languages/dotnet.py index e26b45c..4c3955e 100644 --- a/pre_commit/languages/dotnet.py +++ b/pre_commit/languages/dotnet.py @@ -9,20 +9,18 @@ import zipfile from typing import Generator from typing import Sequence -import pre_commit.constants as C from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.prefix import Prefix -from pre_commit.util import clean_path_on_failure ENVIRONMENT_DIR = 'dotnetenv' BIN_DIR = 'bin' get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check +run_hook = helpers.basic_run_hook def get_env_patch(venv: str) -> PatchesT: @@ -32,9 +30,8 @@ def get_env_patch(venv: str) -> PatchesT: @contextlib.contextmanager -def in_env(prefix: Prefix) -> Generator[None, None, None]: - directory = helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT) - envdir = prefix.path(directory) +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield @@ -63,66 +60,56 @@ def install_environment( helpers.assert_version_default('dotnet', version) helpers.assert_no_additional_deps('dotnet', additional_dependencies) - envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version)) - with clean_path_on_failure(envdir): - build_dir = 'pre-commit-build' - - # Build & pack nupkg file - helpers.run_setup_cmd( - prefix, - ( - 'dotnet', 'pack', - '--configuration', 'Release', - '--output', build_dir, - ), - ) - - nupkg_dir = prefix.path(build_dir) - nupkgs = [x for x in os.listdir(nupkg_dir) if x.endswith('.nupkg')] - - if not nupkgs: - raise AssertionError('could not find any build outputs to install') - - for nupkg in nupkgs: - with zipfile.ZipFile(os.path.join(nupkg_dir, nupkg)) as f: - nuspec, = (x for x in f.namelist() if x.endswith('.nuspec')) - with f.open(nuspec) as spec: - tree = xml.etree.ElementTree.parse(spec) - - namespace = re.match(r'{.*}', tree.getroot().tag) - if not namespace: - raise AssertionError('could not parse namespace from nuspec') - - tool_id_element = tree.find(f'.//{namespace[0]}id') - if tool_id_element is None: - raise AssertionError('expected to find an "id" element') - - tool_id = tool_id_element.text - if not tool_id: - raise AssertionError('"id" element missing tool name') - - # Install to bin dir - with _nuget_config_no_sources() as nuget_config: - helpers.run_setup_cmd( - prefix, - ( - 'dotnet', 'tool', 'install', - '--configfile', nuget_config, - '--tool-path', os.path.join(envdir, BIN_DIR), - '--add-source', build_dir, - tool_id, - ), - ) - - # Clean the git dir, ignoring the environment dir - clean_cmd = ('git', 'clean', '-ffxd', '-e', f'{ENVIRONMENT_DIR}-*') - helpers.run_setup_cmd(prefix, clean_cmd) - - -def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, -) -> tuple[int, bytes]: - with in_env(hook.prefix): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) + build_dir = 'pre-commit-build' + + # Build & pack nupkg file + helpers.run_setup_cmd( + prefix, + ( + 'dotnet', 'pack', + '--configuration', 'Release', + '--output', build_dir, + ), + ) + + nupkg_dir = prefix.path(build_dir) + nupkgs = [x for x in os.listdir(nupkg_dir) if x.endswith('.nupkg')] + + if not nupkgs: + raise AssertionError('could not find any build outputs to install') + + for nupkg in nupkgs: + with zipfile.ZipFile(os.path.join(nupkg_dir, nupkg)) as f: + nuspec, = (x for x in f.namelist() if x.endswith('.nuspec')) + with f.open(nuspec) as spec: + tree = xml.etree.ElementTree.parse(spec) + + namespace = re.match(r'{.*}', tree.getroot().tag) + if not namespace: + raise AssertionError('could not parse namespace from nuspec') + + tool_id_element = tree.find(f'.//{namespace[0]}id') + if tool_id_element is None: + raise AssertionError('expected to find an "id" element') + + tool_id = tool_id_element.text + if not tool_id: + raise AssertionError('"id" element missing tool name') + + # Install to bin dir + with _nuget_config_no_sources() as nuget_config: + helpers.run_setup_cmd( + prefix, + ( + 'dotnet', 'tool', 'install', + '--configfile', nuget_config, + '--tool-path', os.path.join(envdir, BIN_DIR), + '--add-source', build_dir, + tool_id, + ), + ) + + # Clean the git dir, ignoring the environment dir + clean_cmd = ('git', 'clean', '-ffxd', '-e', f'{ENVIRONMENT_DIR}-*') + helpers.run_setup_cmd(prefix, clean_cmd) diff --git a/pre_commit/languages/fail.py b/pre_commit/languages/fail.py index 00b06a9..33df067 100644 --- a/pre_commit/languages/fail.py +++ b/pre_commit/languages/fail.py @@ -2,20 +2,26 @@ from __future__ import annotations from typing import Sequence -from pre_commit.hook import Hook from pre_commit.languages import helpers +from pre_commit.prefix import Prefix ENVIRONMENT_DIR = None get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check install_environment = helpers.no_install +in_env = helpers.no_env def run_hook( - hook: Hook, + prefix: Prefix, + entry: str, + args: Sequence[str], file_args: Sequence[str], + *, + is_local: bool, + require_serial: bool, color: bool, ) -> tuple[int, bytes]: - out = f'{hook.entry}\n\n'.encode() + out = f'{entry}\n\n'.encode() out += b'\n'.join(f.encode() for f in file_args) + b'\n' return 1, out diff --git a/pre_commit/languages/golang.py b/pre_commit/languages/golang.py index a5f9dba..3c4b652 100644 --- a/pre_commit/languages/golang.py +++ b/pre_commit/languages/golang.py @@ -1,58 +1,129 @@ from __future__ import annotations import contextlib +import functools +import json import os.path +import platform +import shutil import sys +import tarfile +import tempfile +import urllib.error +import urllib.request +import zipfile +from typing import ContextManager from typing import Generator +from typing import IO +from typing import Protocol from typing import Sequence import pre_commit.constants as C -from pre_commit import git from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.prefix import Prefix -from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output -from pre_commit.util import cmd_output_b from pre_commit.util import rmtree ENVIRONMENT_DIR = 'golangenv' -get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check +run_hook = helpers.basic_run_hook +_ARCH_ALIASES = { + 'x86_64': 'amd64', + 'i386': '386', + 'aarch64': 'arm64', + 'armv8': 'arm64', + 'armv7l': 'armv6l', +} +_ARCH = platform.machine().lower() +_ARCH = _ARCH_ALIASES.get(_ARCH, _ARCH) + + +class ExtractAll(Protocol): + def extractall(self, path: str) -> None: ... + + +if sys.platform == 'win32': # pragma: win32 cover + _EXT = 'zip' + + def _open_archive(bio: IO[bytes]) -> ContextManager[ExtractAll]: + return zipfile.ZipFile(bio) +else: # pragma: win32 no cover + _EXT = 'tar.gz' + + def _open_archive(bio: IO[bytes]) -> ContextManager[ExtractAll]: + return tarfile.open(fileobj=bio) + + +@functools.lru_cache(maxsize=1) +def get_default_version() -> str: + if helpers.exe_exists('go'): + return 'system' + else: + return C.DEFAULT + + +def get_env_patch(venv: str, version: str) -> PatchesT: + if version == 'system': + return ( + ('PATH', (os.path.join(venv, 'bin'), os.pathsep, Var('PATH'))), + ) -def get_env_patch(venv: str) -> PatchesT: return ( - ('PATH', (os.path.join(venv, 'bin'), os.pathsep, Var('PATH'))), + ('GOROOT', os.path.join(venv, '.go')), + ( + 'PATH', ( + os.path.join(venv, 'bin'), os.pathsep, + os.path.join(venv, '.go', 'bin'), os.pathsep, Var('PATH'), + ), + ), ) -@contextlib.contextmanager -def in_env(prefix: Prefix) -> Generator[None, None, None]: - envdir = prefix.path( - helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), - ) - with envcontext(get_env_patch(envdir)): - yield +@functools.lru_cache +def _infer_go_version(version: str) -> str: + if version != C.DEFAULT: + return version + resp = urllib.request.urlopen('https://go.dev/dl/?mode=json') + # TODO: 3.9+ .removeprefix('go') + return json.load(resp)[0]['version'][2:] -def guess_go_dir(remote_url: str) -> str: - if remote_url.endswith('.git'): - remote_url = remote_url[:-1 * len('.git')] - looks_like_url = ( - not remote_url.startswith('file://') and - ('//' in remote_url or '@' in remote_url) - ) - remote_url = remote_url.replace(':', '/') - if looks_like_url: - _, _, remote_url = remote_url.rpartition('//') - _, _, remote_url = remote_url.rpartition('@') - return remote_url +def _get_url(version: str) -> str: + os_name = platform.system().lower() + version = _infer_go_version(version) + return f'https://dl.google.com/go/go{version}.{os_name}-{_ARCH}.{_EXT}' + + +def _install_go(version: str, dest: str) -> None: + try: + resp = urllib.request.urlopen(_get_url(version)) + except urllib.error.HTTPError as e: # pragma: no cover + if e.code == 404: + raise ValueError( + f'Could not find a version matching your system requirements ' + f'(os={platform.system().lower()}; arch={_ARCH})', + ) from e + else: + raise else: - return 'unknown_src_dir' + with tempfile.TemporaryFile() as f: + shutil.copyfileobj(resp, f) + f.seek(0) + + with _open_archive(f) as archive: + archive.extractall(dest) + shutil.move(os.path.join(dest, 'go'), os.path.join(dest, '.go')) + + +@contextlib.contextmanager +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) + with envcontext(get_env_patch(envdir, version)): + yield def install_environment( @@ -60,42 +131,29 @@ def install_environment( version: str, additional_dependencies: Sequence[str], ) -> None: - helpers.assert_version_default('golang', version) - directory = prefix.path( - helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), - ) + env_dir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) - with clean_path_on_failure(directory): - remote = git.get_remote_url(prefix.prefix_dir) - repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote)) + if version != 'system': + _install_go(version, env_dir) - # Clone into the goenv we'll create - cmd = ('git', 'clone', '--recursive', '.', repo_src_dir) - helpers.run_setup_cmd(prefix, cmd) - - if sys.platform == 'cygwin': # pragma: no cover - _, gopath, _ = cmd_output('cygpath', '-w', directory) - gopath = gopath.strip() - else: - gopath = directory - env = dict(os.environ, GOPATH=gopath) - env.pop('GOBIN', None) - cmd_output_b('go', 'install', './...', cwd=repo_src_dir, env=env) - for dependency in additional_dependencies: - cmd_output_b( - 'go', 'install', dependency, cwd=repo_src_dir, env=env, - ) - # Same some disk space, we don't need these after installation - rmtree(prefix.path(directory, 'src')) - pkgdir = prefix.path(directory, 'pkg') - if os.path.exists(pkgdir): # pragma: no cover (go<1.10) - rmtree(pkgdir) - - -def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, -) -> tuple[int, bytes]: - with in_env(hook.prefix): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) + if sys.platform == 'cygwin': # pragma: no cover + gopath = cmd_output('cygpath', '-w', env_dir)[1].strip() + else: + gopath = env_dir + + env = dict(os.environ, GOPATH=gopath) + env.pop('GOBIN', None) + if version != 'system': + env['GOROOT'] = os.path.join(env_dir, '.go') + env['PATH'] = os.pathsep.join(( + os.path.join(env_dir, '.go', 'bin'), os.environ['PATH'], + )) + + helpers.run_setup_cmd(prefix, ('go', 'install', './...'), env=env) + for dependency in additional_dependencies: + helpers.run_setup_cmd(prefix, ('go', 'install', dependency), env=env) + + # save some disk space -- we don't need this after installation + pkgdir = os.path.join(env_dir, 'pkg') + if os.path.exists(pkgdir): # pragma: no branch (always true on windows?) + rmtree(pkgdir) diff --git a/pre_commit/languages/helpers.py b/pre_commit/languages/helpers.py index 0be08b5..d1be409 100644 --- a/pre_commit/languages/helpers.py +++ b/pre_commit/languages/helpers.py @@ -1,17 +1,18 @@ from __future__ import annotations +import contextlib import multiprocessing import os import random import re +import shlex from typing import Any +from typing import Generator from typing import NoReturn -from typing import overload from typing import Sequence import pre_commit.constants as C from pre_commit import parse_shebang -from pre_commit.hook import Hook from pre_commit.prefix import Prefix from pre_commit.util import cmd_output_b from pre_commit.xargs import xargs @@ -48,17 +49,8 @@ def run_setup_cmd(prefix: Prefix, cmd: tuple[str, ...], **kwargs: Any) -> None: cmd_output_b(*cmd, cwd=prefix.prefix_dir, **kwargs) -@overload -def environment_dir(d: None, language_version: str) -> None: ... -@overload -def environment_dir(d: str, language_version: str) -> str: ... - - -def environment_dir(d: str | None, language_version: str) -> str | None: - if d is None: - return None - else: - return f'{d}-{language_version}' +def environment_dir(prefix: Prefix, d: str, language_version: str) -> str: + return prefix.path(f'{d}-{language_version}') def assert_version_default(binary: str, version: str) -> None: @@ -94,11 +86,16 @@ def no_install( version: str, additional_dependencies: Sequence[str], ) -> NoReturn: - raise AssertionError('This type is not installable') + raise AssertionError('This language is not installable') + +@contextlib.contextmanager +def no_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + yield -def target_concurrency(hook: Hook) -> int: - if hook.require_serial or 'PRE_COMMIT_NO_CONCURRENCY' in os.environ: + +def target_concurrency() -> int: + if 'PRE_COMMIT_NO_CONCURRENCY' in os.environ: return 1 else: # Travis appears to have a bunch of CPUs, but we can't use them all. @@ -122,13 +119,40 @@ def _shuffled(seq: Sequence[str]) -> list[str]: def run_xargs( - hook: Hook, cmd: tuple[str, ...], file_args: Sequence[str], - **kwargs: Any, + *, + require_serial: bool, + color: bool, +) -> tuple[int, bytes]: + if require_serial: + jobs = 1 + else: + # Shuffle the files so that they more evenly fill out the xargs + # partitions, but do it deterministically in case a hook cares about + # ordering. + file_args = _shuffled(file_args) + jobs = target_concurrency() + return xargs(cmd, file_args, target_concurrency=jobs, color=color) + + +def hook_cmd(entry: str, args: Sequence[str]) -> tuple[str, ...]: + return (*shlex.split(entry), *args) + + +def basic_run_hook( + prefix: Prefix, + entry: str, + args: Sequence[str], + file_args: Sequence[str], + *, + is_local: bool, + require_serial: bool, + color: bool, ) -> tuple[int, bytes]: - # Shuffle the files so that they more evenly fill out the xargs partitions, - # but do it deterministically in case a hook cares about ordering. - file_args = _shuffled(file_args) - kwargs['target_concurrency'] = target_concurrency(hook) - return xargs(cmd, file_args, **kwargs) + return run_xargs( + hook_cmd(entry, args), + file_args, + require_serial=require_serial, + color=color, + ) diff --git a/pre_commit/languages/lua.py b/pre_commit/languages/lua.py index 49aa730..ffc40b5 100644 --- a/pre_commit/languages/lua.py +++ b/pre_commit/languages/lua.py @@ -6,19 +6,17 @@ import sys from typing import Generator from typing import Sequence -import pre_commit.constants as C from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.prefix import Prefix -from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output ENVIRONMENT_DIR = 'lua_env' get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check +run_hook = helpers.basic_run_hook def _get_lua_version() -> str: # pragma: win32 no cover @@ -45,14 +43,10 @@ def get_env_patch(d: str) -> PatchesT: # pragma: win32 no cover ) -def _envdir(prefix: Prefix) -> str: # pragma: win32 no cover - directory = helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT) - return prefix.path(directory) - - @contextlib.contextmanager # pragma: win32 no cover -def in_env(prefix: Prefix) -> Generator[None, None, None]: - with envcontext(get_env_patch(_envdir(prefix))): +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) + with envcontext(get_env_patch(envdir)): yield @@ -63,29 +57,19 @@ def install_environment( ) -> None: # pragma: win32 no cover helpers.assert_version_default('lua', version) - envdir = _envdir(prefix) - with clean_path_on_failure(envdir): - with in_env(prefix): - # luarocks doesn't bootstrap a tree prior to installing - # so ensure the directory exists. - os.makedirs(envdir, exist_ok=True) - - # Older luarocks (e.g., 2.4.2) expect the rockspec as an arg - for rockspec in prefix.star('.rockspec'): - make_cmd = ('luarocks', '--tree', envdir, 'make', rockspec) - helpers.run_setup_cmd(prefix, make_cmd) - - # luarocks can't install multiple packages at once - # so install them individually. - for dependency in additional_dependencies: - cmd = ('luarocks', '--tree', envdir, 'install', dependency) - helpers.run_setup_cmd(prefix, cmd) - - -def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, -) -> tuple[int, bytes]: # pragma: win32 no cover - with in_env(hook.prefix): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) + with in_env(prefix, version): + # luarocks doesn't bootstrap a tree prior to installing + # so ensure the directory exists. + os.makedirs(envdir, exist_ok=True) + + # Older luarocks (e.g., 2.4.2) expect the rockspec as an arg + for rockspec in prefix.star('.rockspec'): + make_cmd = ('luarocks', '--tree', envdir, 'make', rockspec) + helpers.run_setup_cmd(prefix, make_cmd) + + # luarocks can't install multiple packages at once + # so install them individually. + for dependency in additional_dependencies: + cmd = ('luarocks', '--tree', envdir, 'install', dependency) + helpers.run_setup_cmd(prefix, cmd) diff --git a/pre_commit/languages/node.py b/pre_commit/languages/node.py index 37a5b63..9688da3 100644 --- a/pre_commit/languages/node.py +++ b/pre_commit/languages/node.py @@ -12,16 +12,15 @@ from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import UNSET from pre_commit.envcontext import Var -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.languages.python import bin_dir from pre_commit.prefix import Prefix -from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output from pre_commit.util import cmd_output_b from pre_commit.util import rmtree ENVIRONMENT_DIR = 'node_env' +run_hook = helpers.basic_run_hook @functools.lru_cache(maxsize=1) @@ -37,11 +36,6 @@ def get_default_version() -> str: return C.DEFAULT -def _envdir(prefix: Prefix, version: str) -> str: - directory = helpers.environment_dir(ENVIRONMENT_DIR, version) - return prefix.path(directory) - - def get_env_patch(venv: str) -> PatchesT: if sys.platform == 'cygwin': # pragma: no cover _, win_venv, _ = cmd_output('cygpath', '-w', venv) @@ -65,11 +59,9 @@ def get_env_patch(venv: str) -> PatchesT: @contextlib.contextmanager -def in_env( - prefix: Prefix, - language_version: str, -) -> Generator[None, None, None]: - with envcontext(get_env_patch(_envdir(prefix, language_version))): +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) + with envcontext(get_env_patch(envdir)): yield @@ -85,47 +77,34 @@ def health_check(prefix: Prefix, language_version: str) -> str | None: def install_environment( prefix: Prefix, version: str, additional_dependencies: Sequence[str], ) -> None: - additional_dependencies = tuple(additional_dependencies) assert prefix.exists('package.json') - envdir = _envdir(prefix, version) + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx?f=255&MSPPError=-2147217396#maxpath if sys.platform == 'win32': # pragma: no cover envdir = fr'\\?\{os.path.normpath(envdir)}' - with clean_path_on_failure(envdir): - cmd = [ - sys.executable, '-mnodeenv', '--prebuilt', '--clean-src', envdir, - ] - if version != C.DEFAULT: - cmd.extend(['-n', version]) - cmd_output_b(*cmd) - - with in_env(prefix, version): - # https://npm.community/t/npm-install-g-git-vs-git-clone-cd-npm-install-g/5449 - # install as if we installed from git - - local_install_cmd = ( - 'npm', 'install', '--dev', '--prod', - '--ignore-prepublish', '--no-progress', '--no-save', - ) - helpers.run_setup_cmd(prefix, local_install_cmd) - - _, pkg, _ = cmd_output('npm', 'pack', cwd=prefix.prefix_dir) - pkg = prefix.path(pkg.strip()) - - install = ('npm', 'install', '-g', pkg, *additional_dependencies) - helpers.run_setup_cmd(prefix, install) - - # clean these up after installation - if prefix.exists('node_modules'): # pragma: win32 no cover - rmtree(prefix.path('node_modules')) - os.remove(pkg) - - -def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, -) -> tuple[int, bytes]: - with in_env(hook.prefix, hook.language_version): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) + cmd = [sys.executable, '-mnodeenv', '--prebuilt', '--clean-src', envdir] + if version != C.DEFAULT: + cmd.extend(['-n', version]) + cmd_output_b(*cmd) + + with in_env(prefix, version): + # https://npm.community/t/npm-install-g-git-vs-git-clone-cd-npm-install-g/5449 + # install as if we installed from git + + local_install_cmd = ( + 'npm', 'install', '--dev', '--prod', + '--ignore-prepublish', '--no-progress', '--no-save', + ) + helpers.run_setup_cmd(prefix, local_install_cmd) + + _, pkg, _ = cmd_output('npm', 'pack', cwd=prefix.prefix_dir) + pkg = prefix.path(pkg.strip()) + + install = ('npm', 'install', '-g', pkg, *additional_dependencies) + helpers.run_setup_cmd(prefix, install) + + # clean these up after installation + if prefix.exists('node_modules'): # pragma: win32 no cover + rmtree(prefix.path('node_modules')) + os.remove(pkg) diff --git a/pre_commit/languages/perl.py b/pre_commit/languages/perl.py index 78bd65a..2530c0e 100644 --- a/pre_commit/languages/perl.py +++ b/pre_commit/languages/perl.py @@ -9,19 +9,13 @@ from typing import Sequence from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.prefix import Prefix -from pre_commit.util import clean_path_on_failure ENVIRONMENT_DIR = 'perl_env' get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check - - -def _envdir(prefix: Prefix, version: str) -> str: - directory = helpers.environment_dir(ENVIRONMENT_DIR, version) - return prefix.path(directory) +run_hook = helpers.basic_run_hook def get_env_patch(venv: str) -> PatchesT: @@ -39,11 +33,9 @@ def get_env_patch(venv: str) -> PatchesT: @contextlib.contextmanager -def in_env( - prefix: Prefix, - language_version: str, -) -> Generator[None, None, None]: - with envcontext(get_env_patch(_envdir(prefix, language_version))): +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) + with envcontext(get_env_patch(envdir)): yield @@ -52,17 +44,7 @@ def install_environment( ) -> None: helpers.assert_version_default('perl', version) - with clean_path_on_failure(_envdir(prefix, version)): - with in_env(prefix, version): - helpers.run_setup_cmd( - prefix, ('cpan', '-T', '.', *additional_dependencies), - ) - - -def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, -) -> tuple[int, bytes]: - with in_env(hook.prefix, hook.language_version): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) + with in_env(prefix, version): + helpers.run_setup_cmd( + prefix, ('cpan', '-T', '.', *additional_dependencies), + ) diff --git a/pre_commit/languages/pygrep.py b/pre_commit/languages/pygrep.py index 2e2072b..f0eb9a9 100644 --- a/pre_commit/languages/pygrep.py +++ b/pre_commit/languages/pygrep.py @@ -8,14 +8,15 @@ from typing import Pattern from typing import Sequence from pre_commit import output -from pre_commit.hook import Hook from pre_commit.languages import helpers +from pre_commit.prefix import Prefix from pre_commit.xargs import xargs ENVIRONMENT_DIR = None get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check install_environment = helpers.no_install +in_env = helpers.no_env def _process_filename_by_line(pattern: Pattern[bytes], filename: str) -> int: @@ -87,12 +88,17 @@ FNS = { def run_hook( - hook: Hook, + prefix: Prefix, + entry: str, + args: Sequence[str], file_args: Sequence[str], + *, + is_local: bool, + require_serial: bool, color: bool, ) -> tuple[int, bytes]: - exe = (sys.executable, '-m', __name__) + tuple(hook.args) + (hook.entry,) - return xargs(exe, file_args, color=color) + cmd = (sys.executable, '-m', __name__, *args, entry) + return xargs(cmd, file_args, color=color) def main(argv: Sequence[str] | None = None) -> int: diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py index 19fa247..c373646 100644 --- a/pre_commit/languages/python.py +++ b/pre_commit/languages/python.py @@ -12,17 +12,16 @@ from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import UNSET from pre_commit.envcontext import Var -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.parse_shebang import find_executable from pre_commit.prefix import Prefix from pre_commit.util import CalledProcessError -from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output from pre_commit.util import cmd_output_b from pre_commit.util import win_exe ENVIRONMENT_DIR = 'py_env' +run_hook = helpers.basic_run_hook @functools.lru_cache(maxsize=None) @@ -153,19 +152,14 @@ def norm_version(version: str) -> str | None: @contextlib.contextmanager -def in_env( - prefix: Prefix, - language_version: str, -) -> Generator[None, None, None]: - directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version) - envdir = prefix.path(directory) +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield def health_check(prefix: Prefix, language_version: str) -> str | None: - directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version) - envdir = prefix.path(directory) + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, language_version) pyvenv_cfg = os.path.join(envdir, 'pyvenv.cfg') # created with "old" virtualenv @@ -208,23 +202,13 @@ def install_environment( version: str, additional_dependencies: Sequence[str], ) -> None: - envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version)) + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) venv_cmd = [sys.executable, '-mvirtualenv', envdir] python = norm_version(version) if python is not None: venv_cmd.extend(('-p', python)) install_cmd = ('python', '-mpip', 'install', '.', *additional_dependencies) - with clean_path_on_failure(envdir): - cmd_output_b(*venv_cmd, cwd='/') - with in_env(prefix, version): - helpers.run_setup_cmd(prefix, install_cmd) - - -def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, -) -> tuple[int, bytes]: - with in_env(hook.prefix, hook.language_version): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) + cmd_output_b(*venv_cmd, cwd='/') + with in_env(prefix, version): + helpers.run_setup_cmd(prefix, install_cmd) diff --git a/pre_commit/languages/r.py b/pre_commit/languages/r.py index d281102..e238365 100644 --- a/pre_commit/languages/r.py +++ b/pre_commit/languages/r.py @@ -10,10 +10,8 @@ from typing import Sequence from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import UNSET -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.prefix import Prefix -from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output_b from pre_commit.util import win_exe @@ -31,32 +29,22 @@ def get_env_patch(venv: str) -> PatchesT: @contextlib.contextmanager -def in_env( - prefix: Prefix, - language_version: str, -) -> Generator[None, None, None]: - envdir = _get_env_dir(prefix, language_version) +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield -def _get_env_dir(prefix: Prefix, version: str) -> str: - return prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version)) - - -def _prefix_if_non_local_file_entry( - entry: Sequence[str], - prefix: Prefix, - src: str, +def _prefix_if_file_entry( + entry: list[str], + prefix: Prefix, + *, + is_local: bool, ) -> Sequence[str]: - if entry[1] == '-e': + if entry[1] == '-e' or is_local: return entry[1:] else: - if src == 'local': - path = entry[1] - else: - path = prefix.path(entry[1]) - return (path,) + return (prefix.path(entry[1]),) def _rscript_exec() -> str: @@ -67,7 +55,7 @@ def _rscript_exec() -> str: return os.path.join(r_home, 'bin', win_exe('Rscript')) -def _entry_validate(entry: Sequence[str]) -> None: +def _entry_validate(entry: list[str]) -> None: """ Allowed entries: # Rscript -e expr @@ -81,20 +69,23 @@ def _entry_validate(entry: Sequence[str]) -> None: raise ValueError('You can supply at most one expression.') elif len(entry) > 2: raise ValueError( - 'The only valid syntax is `Rscript -e {expr}`', + 'The only valid syntax is `Rscript -e {expr}`' 'or `Rscript path/to/hook/script`', ) -def _cmd_from_hook(hook: Hook) -> tuple[str, ...]: - entry = shlex.split(hook.entry) - _entry_validate(entry) +def _cmd_from_hook( + prefix: Prefix, + entry: str, + args: Sequence[str], + *, + is_local: bool, +) -> tuple[str, ...]: + cmd = shlex.split(entry) + _entry_validate(cmd) - return ( - *entry[:1], *RSCRIPT_OPTS, - *_prefix_if_non_local_file_entry(entry, hook.prefix, hook.src), - *hook.args, - ) + cmd_part = _prefix_if_file_entry(cmd, prefix, is_local=is_local) + return (cmd[0], *RSCRIPT_OPTS, *cmd_part, *args) def install_environment( @@ -102,55 +93,54 @@ def install_environment( version: str, additional_dependencies: Sequence[str], ) -> None: - env_dir = _get_env_dir(prefix, version) - with clean_path_on_failure(env_dir): - os.makedirs(env_dir, exist_ok=True) - shutil.copy(prefix.path('renv.lock'), env_dir) - shutil.copytree(prefix.path('renv'), os.path.join(env_dir, 'renv')) - - r_code_inst_environment = f"""\ - prefix_dir <- {prefix.prefix_dir!r} - options( - repos = c(CRAN = "https://cran.rstudio.com"), - renv.consent = TRUE - ) - source("renv/activate.R") - renv::restore() - activate_statement <- paste0( - 'suppressWarnings({{', - 'old <- setwd("', getwd(), '"); ', - 'source("renv/activate.R"); ', - 'setwd(old); ', - 'renv::load("', getwd(), '");}})' - ) - writeLines(activate_statement, 'activate.R') - is_package <- tryCatch( - {{ - path_desc <- file.path(prefix_dir, 'DESCRIPTION') - suppressWarnings(desc <- read.dcf(path_desc)) - "Package" %in% colnames(desc) - }}, - error = function(...) FALSE - ) - if (is_package) {{ - renv::install(prefix_dir) - }} - """ - - cmd_output_b( - _rscript_exec(), '--vanilla', '-e', - _inline_r_setup(r_code_inst_environment), - cwd=env_dir, + env_dir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) + os.makedirs(env_dir, exist_ok=True) + shutil.copy(prefix.path('renv.lock'), env_dir) + shutil.copytree(prefix.path('renv'), os.path.join(env_dir, 'renv')) + + r_code_inst_environment = f"""\ + prefix_dir <- {prefix.prefix_dir!r} + options( + repos = c(CRAN = "https://cran.rstudio.com"), + renv.consent = TRUE + ) + source("renv/activate.R") + renv::restore() + activate_statement <- paste0( + 'suppressWarnings({{', + 'old <- setwd("', getwd(), '"); ', + 'source("renv/activate.R"); ', + 'setwd(old); ', + 'renv::load("', getwd(), '");}})' ) - if additional_dependencies: - r_code_inst_add = 'renv::install(commandArgs(trailingOnly = TRUE))' - with in_env(prefix, version): - cmd_output_b( - _rscript_exec(), *RSCRIPT_OPTS, '-e', - _inline_r_setup(r_code_inst_add), - *additional_dependencies, - cwd=env_dir, - ) + writeLines(activate_statement, 'activate.R') + is_package <- tryCatch( + {{ + path_desc <- file.path(prefix_dir, 'DESCRIPTION') + suppressWarnings(desc <- read.dcf(path_desc)) + "Package" %in% colnames(desc) + }}, + error = function(...) FALSE + ) + if (is_package) {{ + renv::install(prefix_dir) + }} + """ + + cmd_output_b( + _rscript_exec(), '--vanilla', '-e', + _inline_r_setup(r_code_inst_environment), + cwd=env_dir, + ) + if additional_dependencies: + r_code_inst_add = 'renv::install(commandArgs(trailingOnly = TRUE))' + with in_env(prefix, version): + cmd_output_b( + _rscript_exec(), *RSCRIPT_OPTS, '-e', + _inline_r_setup(r_code_inst_add), + *additional_dependencies, + cwd=env_dir, + ) def _inline_r_setup(code: str) -> str: @@ -166,11 +156,19 @@ def _inline_r_setup(code: str) -> str: def run_hook( - hook: Hook, + prefix: Prefix, + entry: str, + args: Sequence[str], file_args: Sequence[str], + *, + is_local: bool, + require_serial: bool, color: bool, ) -> tuple[int, bytes]: - with in_env(hook.prefix, hook.language_version): - return helpers.run_xargs( - hook, _cmd_from_hook(hook), file_args, color=color, - ) + cmd = _cmd_from_hook(prefix, entry, args, is_local=is_local) + return helpers.run_xargs( + cmd, + file_args, + require_serial=require_serial, + color=color, + ) diff --git a/pre_commit/languages/ruby.py b/pre_commit/languages/ruby.py index 8955dd0..b4d4b45 100644 --- a/pre_commit/languages/ruby.py +++ b/pre_commit/languages/ruby.py @@ -13,15 +13,14 @@ from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import UNSET from pre_commit.envcontext import Var -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.prefix import Prefix from pre_commit.util import CalledProcessError -from pre_commit.util import clean_path_on_failure from pre_commit.util import resource_bytesio ENVIRONMENT_DIR = 'rbenv' health_check = helpers.basic_health_check +run_hook = helpers.basic_run_hook @functools.lru_cache(maxsize=1) @@ -40,6 +39,7 @@ def get_env_patch( ('GEM_HOME', os.path.join(venv, 'gems')), ('GEM_PATH', UNSET), ('BUNDLE_IGNORE_CONFIG', '1'), + ('BUNDLE_GEMFILE', os.devnull), ) if language_version == 'system': patches += ( @@ -68,14 +68,9 @@ def get_env_patch( @contextlib.contextmanager -def in_env( - prefix: Prefix, - language_version: str, -) -> Generator[None, None, None]: - envdir = prefix.path( - helpers.environment_dir(ENVIRONMENT_DIR, language_version), - ) - with envcontext(get_env_patch(envdir, language_version)): +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) + with envcontext(get_env_patch(envdir, version)): yield @@ -89,14 +84,14 @@ def _install_rbenv( prefix: Prefix, version: str, ) -> None: # pragma: win32 no cover - directory = helpers.environment_dir(ENVIRONMENT_DIR, version) + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) _extract_resource('rbenv.tar.gz', prefix.path('.')) - shutil.move(prefix.path('rbenv'), prefix.path(directory)) + shutil.move(prefix.path('rbenv'), envdir) # Only install ruby-build if the version is specified if version != C.DEFAULT: - plugins_dir = prefix.path(directory, 'plugins') + plugins_dir = os.path.join(envdir, 'plugins') _extract_resource('ruby-download.tar.gz', plugins_dir) _extract_resource('ruby-build.tar.gz', plugins_dir) @@ -115,39 +110,27 @@ def _install_ruby( def install_environment( prefix: Prefix, version: str, additional_dependencies: Sequence[str], ) -> None: - additional_dependencies = tuple(additional_dependencies) - directory = helpers.environment_dir(ENVIRONMENT_DIR, version) - with clean_path_on_failure(prefix.path(directory)): - if version != 'system': # pragma: win32 no cover - _install_rbenv(prefix, version) - with in_env(prefix, version): - # Need to call this before installing so rbenv's directories - # are set up - helpers.run_setup_cmd(prefix, ('rbenv', 'init', '-')) - if version != C.DEFAULT: - _install_ruby(prefix, version) - # Need to call this after installing to set up the shims - helpers.run_setup_cmd(prefix, ('rbenv', 'rehash')) - + if version != 'system': # pragma: win32 no cover + _install_rbenv(prefix, version) with in_env(prefix, version): - helpers.run_setup_cmd( - prefix, ('gem', 'build', *prefix.star('.gemspec')), - ) - helpers.run_setup_cmd( - prefix, - ( - 'gem', 'install', - '--no-document', '--no-format-executable', - '--no-user-install', - *prefix.star('.gem'), *additional_dependencies, - ), - ) - - -def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, -) -> tuple[int, bytes]: - with in_env(hook.prefix, hook.language_version): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) + # Need to call this before installing so rbenv's directories + # are set up + helpers.run_setup_cmd(prefix, ('rbenv', 'init', '-')) + if version != C.DEFAULT: + _install_ruby(prefix, version) + # Need to call this after installing to set up the shims + helpers.run_setup_cmd(prefix, ('rbenv', 'rehash')) + + with in_env(prefix, version): + helpers.run_setup_cmd( + prefix, ('gem', 'build', *prefix.star('.gemspec')), + ) + helpers.run_setup_cmd( + prefix, + ( + 'gem', 'install', + '--no-document', '--no-format-executable', + '--no-user-install', + *prefix.star('.gem'), *additional_dependencies, + ), + ) diff --git a/pre_commit/languages/rust.py b/pre_commit/languages/rust.py index 204f2aa..391fd86 100644 --- a/pre_commit/languages/rust.py +++ b/pre_commit/languages/rust.py @@ -15,16 +15,15 @@ from pre_commit import parse_shebang from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.prefix import Prefix -from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output_b from pre_commit.util import make_executable from pre_commit.util import win_exe ENVIRONMENT_DIR = 'rustenv' health_check = helpers.basic_health_check +run_hook = helpers.basic_run_hook @functools.lru_cache(maxsize=1) @@ -49,11 +48,6 @@ def _rust_toolchain(language_version: str) -> str: return language_version -def _envdir(prefix: Prefix, version: str) -> str: - directory = helpers.environment_dir(ENVIRONMENT_DIR, version) - return prefix.path(directory) - - def get_env_patch(target_dir: str, version: str) -> PatchesT: return ( ('CARGO_HOME', target_dir), @@ -68,13 +62,9 @@ def get_env_patch(target_dir: str, version: str) -> PatchesT: @contextlib.contextmanager -def in_env( - prefix: Prefix, - language_version: str, -) -> Generator[None, None, None]: - with envcontext( - get_env_patch(_envdir(prefix, language_version), language_version), - ): +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) + with envcontext(get_env_patch(envdir, version)): yield @@ -126,7 +116,7 @@ def install_environment( version: str, additional_dependencies: Sequence[str], ) -> None: - directory = _envdir(prefix, version) + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) # There are two cases where we might want to specify more dependencies: # as dependencies for the library being built, and as binary packages @@ -143,34 +133,24 @@ def install_environment( } lib_deps = set(additional_dependencies) - cli_deps - with clean_path_on_failure(directory): - packages_to_install: set[tuple[str, ...]] = {('--path', '.')} - for cli_dep in cli_deps: - cli_dep = cli_dep[len('cli:'):] - package, _, crate_version = cli_dep.partition(':') - if crate_version != '': - packages_to_install.add((package, '--version', crate_version)) - else: - packages_to_install.add((package,)) - - with in_env(prefix, version): - if version != 'system': - install_rust_with_toolchain(_rust_toolchain(version)) - - if len(lib_deps) > 0: - _add_dependencies(prefix, lib_deps) + packages_to_install: set[tuple[str, ...]] = {('--path', '.')} + for cli_dep in cli_deps: + cli_dep = cli_dep[len('cli:'):] + package, _, crate_version = cli_dep.partition(':') + if crate_version != '': + packages_to_install.add((package, '--version', crate_version)) + else: + packages_to_install.add((package,)) - for args in packages_to_install: - cmd_output_b( - 'cargo', 'install', '--bins', '--root', directory, *args, - cwd=prefix.prefix_dir, - ) + with in_env(prefix, version): + if version != 'system': + install_rust_with_toolchain(_rust_toolchain(version)) + if len(lib_deps) > 0: + _add_dependencies(prefix, lib_deps) -def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, -) -> tuple[int, bytes]: - with in_env(hook.prefix, hook.language_version): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) + for args in packages_to_install: + cmd_output_b( + 'cargo', 'install', '--bins', '--root', envdir, *args, + cwd=prefix.prefix_dir, + ) diff --git a/pre_commit/languages/script.py b/pre_commit/languages/script.py index d5e7677..08325f4 100644 --- a/pre_commit/languages/script.py +++ b/pre_commit/languages/script.py @@ -2,19 +2,31 @@ from __future__ import annotations from typing import Sequence -from pre_commit.hook import Hook from pre_commit.languages import helpers +from pre_commit.prefix import Prefix ENVIRONMENT_DIR = None get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check install_environment = helpers.no_install +in_env = helpers.no_env def run_hook( - hook: Hook, + prefix: Prefix, + entry: str, + args: Sequence[str], file_args: Sequence[str], + *, + is_local: bool, + require_serial: bool, color: bool, ) -> tuple[int, bytes]: - cmd = (hook.prefix.path(hook.cmd[0]), *hook.cmd[1:]) - return helpers.run_xargs(hook, cmd, file_args, color=color) + cmd = helpers.hook_cmd(entry, args) + cmd = (prefix.path(cmd[0]), *cmd[1:]) + return helpers.run_xargs( + cmd, + file_args, + require_serial=require_serial, + color=color, + ) diff --git a/pre_commit/languages/swift.py b/pre_commit/languages/swift.py index 4c68703..c66ad5f 100644 --- a/pre_commit/languages/swift.py +++ b/pre_commit/languages/swift.py @@ -5,21 +5,20 @@ import os from typing import Generator from typing import Sequence -import pre_commit.constants as C from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var -from pre_commit.hook import Hook from pre_commit.languages import helpers from pre_commit.prefix import Prefix -from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output_b +BUILD_DIR = '.build' +BUILD_CONFIG = 'release' + ENVIRONMENT_DIR = 'swift_env' get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check -BUILD_DIR = '.build' -BUILD_CONFIG = 'release' +run_hook = helpers.basic_run_hook def get_env_patch(venv: str) -> PatchesT: # pragma: win32 no cover @@ -28,10 +27,8 @@ def get_env_patch(venv: str) -> PatchesT: # pragma: win32 no cover @contextlib.contextmanager # pragma: win32 no cover -def in_env(prefix: Prefix) -> Generator[None, None, None]: - envdir = prefix.path( - helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), - ) +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield @@ -41,25 +38,13 @@ def install_environment( ) -> None: # pragma: win32 no cover helpers.assert_version_default('swift', version) helpers.assert_no_additional_deps('swift', additional_dependencies) - directory = prefix.path( - helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), - ) + envdir = helpers.environment_dir(prefix, ENVIRONMENT_DIR, version) # Build the swift package - with clean_path_on_failure(directory): - os.mkdir(directory) - cmd_output_b( - 'swift', 'build', - '-C', prefix.prefix_dir, - '-c', BUILD_CONFIG, - '--build-path', os.path.join(directory, BUILD_DIR), - ) - - -def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, -) -> tuple[int, bytes]: # pragma: win32 no cover - with in_env(hook.prefix): - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) + os.mkdir(envdir) + cmd_output_b( + 'swift', 'build', + '-C', prefix.prefix_dir, + '-c', BUILD_CONFIG, + '--build-path', os.path.join(envdir, BUILD_DIR), + ) diff --git a/pre_commit/languages/system.py b/pre_commit/languages/system.py index c64fb36..204cad7 100644 --- a/pre_commit/languages/system.py +++ b/pre_commit/languages/system.py @@ -1,20 +1,10 @@ from __future__ import annotations -from typing import Sequence - -from pre_commit.hook import Hook from pre_commit.languages import helpers - ENVIRONMENT_DIR = None get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check install_environment = helpers.no_install - - -def run_hook( - hook: Hook, - file_args: Sequence[str], - color: bool, -) -> tuple[int, bytes]: - return helpers.run_xargs(hook, hook.cmd, file_args, color=color) +in_env = helpers.no_env +run_hook = helpers.basic_run_hook diff --git a/pre_commit/parse_shebang.py b/pre_commit/parse_shebang.py index 3ac933c..3ee04e8 100644 --- a/pre_commit/parse_shebang.py +++ b/pre_commit/parse_shebang.py @@ -20,13 +20,13 @@ def parse_filename(filename: str) -> tuple[str, ...]: def find_executable( - exe: str, _environ: Mapping[str, str] | None = None, + exe: str, *, env: Mapping[str, str] | None = None, ) -> str | None: exe = os.path.normpath(exe) if os.sep in exe: return exe - environ = _environ if _environ is not None else os.environ + environ = env if env is not None else os.environ if 'PATHEXT' in environ: exts = environ['PATHEXT'].split(os.pathsep) @@ -43,12 +43,12 @@ def find_executable( return None -def normexe(orig: str) -> str: +def normexe(orig: str, *, env: Mapping[str, str] | None = None) -> str: def _error(msg: str) -> NoReturn: raise ExecutableNotFoundError(f'Executable `{orig}` {msg}') if os.sep not in orig and (not os.altsep or os.altsep not in orig): - exe = find_executable(orig) + exe = find_executable(orig, env=env) if exe is None: _error('not found') return exe @@ -62,7 +62,11 @@ def normexe(orig: str) -> str: return orig -def normalize_cmd(cmd: tuple[str, ...]) -> tuple[str, ...]: +def normalize_cmd( + cmd: tuple[str, ...], + *, + env: Mapping[str, str] | None = None, +) -> tuple[str, ...]: """Fixes for the following issues on windows - https://bugs.python.org/issue8557 - windows does not parse shebangs @@ -70,12 +74,12 @@ def normalize_cmd(cmd: tuple[str, ...]) -> tuple[str, ...]: This function also makes deep-path shebangs work just fine """ # Use PATH to determine the executable - exe = normexe(cmd[0]) + exe = normexe(cmd[0], env=env) # Figure out the shebang from the resulting command cmd = parse_filename(exe) + (exe,) + cmd[1:] # This could have given us back another bare executable - exe = normexe(cmd[0]) + exe = normexe(cmd[0], env=env) return (exe,) + cmd[1:] diff --git a/pre_commit/repository.py b/pre_commit/repository.py index 4092277..616faf5 100644 --- a/pre_commit/repository.py +++ b/pre_commit/repository.py @@ -10,28 +10,33 @@ import pre_commit.constants as C from pre_commit.clientlib import load_manifest from pre_commit.clientlib import LOCAL from pre_commit.clientlib import META +from pre_commit.clientlib import parse_version from pre_commit.hook import Hook from pre_commit.languages.all import languages from pre_commit.languages.helpers import environment_dir from pre_commit.prefix import Prefix from pre_commit.store import Store -from pre_commit.util import parse_version +from pre_commit.util import clean_path_on_failure from pre_commit.util import rmtree logger = logging.getLogger('pre_commit') -def _state(additional_deps: Sequence[str]) -> object: - return {'additional_dependencies': sorted(additional_deps)} +def _state_filename_v1(venv: str) -> str: + return os.path.join(venv, '.install_state_v1') + +def _state_filename_v2(venv: str) -> str: + return os.path.join(venv, '.install_state_v2') -def _state_filename(prefix: Prefix, venv: str) -> str: - return prefix.path(venv, f'.install_state_v{C.INSTALLED_STATE_VERSION}') +def _state(additional_deps: Sequence[str]) -> object: + return {'additional_dependencies': sorted(additional_deps)} -def _read_state(prefix: Prefix, venv: str) -> object | None: - filename = _state_filename(prefix, venv) + +def _read_state(venv: str) -> object | None: + filename = _state_filename_v1(venv) if not os.path.exists(filename): return None else: @@ -39,26 +44,22 @@ def _read_state(prefix: Prefix, venv: str) -> object | None: return json.load(f) -def _write_state(prefix: Prefix, venv: str, state: object) -> None: - state_filename = _state_filename(prefix, venv) - staging = f'{state_filename}staging' - with open(staging, 'w') as state_file: - state_file.write(json.dumps(state)) - # Move the file into place atomically to indicate we've installed - os.replace(staging, state_filename) - - def _hook_installed(hook: Hook) -> bool: lang = languages[hook.language] - venv = environment_dir(lang.ENVIRONMENT_DIR, hook.language_version) + if lang.ENVIRONMENT_DIR is None: + return True + + venv = environment_dir( + hook.prefix, + lang.ENVIRONMENT_DIR, + hook.language_version, + ) return ( - venv is None or ( - ( - _read_state(hook.prefix, venv) == - _state(hook.additional_dependencies) - ) and - not lang.health_check(hook.prefix, hook.language_version) - ) + ( + os.path.exists(_state_filename_v2(venv)) or + _read_state(venv) == _state(hook.additional_dependencies) + ) and + not lang.health_check(hook.prefix, hook.language_version) ) @@ -69,26 +70,41 @@ def _hook_install(hook: Hook) -> None: lang = languages[hook.language] assert lang.ENVIRONMENT_DIR is not None - venv = environment_dir(lang.ENVIRONMENT_DIR, hook.language_version) + + venv = environment_dir( + hook.prefix, + lang.ENVIRONMENT_DIR, + hook.language_version, + ) # There's potentially incomplete cleanup from previous runs # Clean it up! - if hook.prefix.exists(venv): - rmtree(hook.prefix.path(venv)) + if os.path.exists(venv): + rmtree(venv) - lang.install_environment( - hook.prefix, hook.language_version, hook.additional_dependencies, - ) - health_error = lang.health_check(hook.prefix, hook.language_version) - if health_error: - raise AssertionError( - f'BUG: expected environment for {hook.language} to be healthy ' - f'immediately after install, please open an issue describing ' - f'your environment\n\n' - f'more info:\n\n{health_error}', + with clean_path_on_failure(venv): + lang.install_environment( + hook.prefix, hook.language_version, hook.additional_dependencies, ) - # Write our state to indicate we're installed - _write_state(hook.prefix, venv, _state(hook.additional_dependencies)) + health_error = lang.health_check(hook.prefix, hook.language_version) + if health_error: + raise AssertionError( + f'BUG: expected environment for {hook.language} to be healthy ' + f'immediately after install, please open an issue describing ' + f'your environment\n\n' + f'more info:\n\n{health_error}', + ) + + # TODO: remove v1 state writing, no longer needed after pre-commit 3.0 + # Write our state to indicate we're installed + state_filename = _state_filename_v1(venv) + staging = f'{state_filename}staging' + with open(staging, 'w') as state_file: + state_file.write(json.dumps(_state(hook.additional_dependencies))) + # Move the file into place atomically to indicate we've installed + os.replace(staging, state_filename) + + open(_state_filename_v2(venv), 'a+').close() def _hook( diff --git a/pre_commit/resources/ruby-build.tar.gz b/pre_commit/resources/ruby-build.tar.gz Binary files differindex 35419f6..b6eacf5 100644 --- a/pre_commit/resources/ruby-build.tar.gz +++ b/pre_commit/resources/ruby-build.tar.gz diff --git a/pre_commit/store.py b/pre_commit/store.py index effebfb..6ddc7c4 100644 --- a/pre_commit/store.py +++ b/pre_commit/store.py @@ -36,6 +36,26 @@ def _get_default_directory() -> str: return os.path.realpath(ret) +_LOCAL_RESOURCES = ( + 'Cargo.toml', 'main.go', 'go.mod', 'main.rs', '.npmignore', + 'package.json', 'pre-commit-package-dev-1.rockspec', + 'pre_commit_placeholder_package.gemspec', 'setup.py', + 'environment.yml', 'Makefile.PL', 'pubspec.yaml', + 'renv.lock', 'renv/activate.R', 'renv/LICENSE.renv', +) + + +def _make_local_repo(directory: str) -> None: + for resource in _LOCAL_RESOURCES: + resource_dirname, resource_basename = os.path.split(resource) + contents = resource_text(f'empty_template_{resource_basename}') + target_dir = os.path.join(directory, resource_dirname) + target_file = os.path.join(target_dir, resource_basename) + os.makedirs(target_dir, exist_ok=True) + with open(target_file, 'w') as f: + f.write(contents) + + class Store: get_default_directory = staticmethod(_get_default_directory) @@ -185,37 +205,9 @@ class Store: return self._new_repo(repo, ref, deps, clone_strategy) - LOCAL_RESOURCES = ( - 'Cargo.toml', 'main.go', 'go.mod', 'main.rs', '.npmignore', - 'package.json', 'pre-commit-package-dev-1.rockspec', - 'pre_commit_placeholder_package.gemspec', 'setup.py', - 'environment.yml', 'Makefile.PL', 'pubspec.yaml', - 'renv.lock', 'renv/activate.R', 'renv/LICENSE.renv', - ) - def make_local(self, deps: Sequence[str]) -> str: - def make_local_strategy(directory: str) -> None: - for resource in self.LOCAL_RESOURCES: - resource_dirname, resource_basename = os.path.split(resource) - contents = resource_text(f'empty_template_{resource_basename}') - target_dir = os.path.join(directory, resource_dirname) - target_file = os.path.join(target_dir, resource_basename) - os.makedirs(target_dir, exist_ok=True) - with open(target_file, 'w') as f: - f.write(contents) - - env = git.no_git_env() - - # initialize the git repository so it looks more like cloned repos - def _git_cmd(*args: str) -> None: - cmd_output_b('git', *args, cwd=directory, env=env) - - git.init_repo(directory, '<<unknown>>') - _git_cmd('add', '.') - git.commit(repo=directory) - return self._new_repo( - 'local', C.LOCAL_REPO_VERSION, deps, make_local_strategy, + 'local', C.LOCAL_REPO_VERSION, deps, _make_local_repo, ) def _create_config_table(self, db: sqlite3.Connection) -> None: diff --git a/pre_commit/util.py b/pre_commit/util.py index b850768..8ea4844 100644 --- a/pre_commit/util.py +++ b/pre_commit/util.py @@ -2,36 +2,20 @@ from __future__ import annotations import contextlib import errno -import functools import importlib.resources import os.path import shutil import stat import subprocess import sys -import tempfile from types import TracebackType from typing import Any from typing import Callable from typing import Generator from typing import IO -import yaml - from pre_commit import parse_shebang -Loader = getattr(yaml, 'CSafeLoader', yaml.SafeLoader) -yaml_load = functools.partial(yaml.load, Loader=Loader) -Dumper = getattr(yaml, 'CSafeDumper', yaml.SafeDumper) - - -def yaml_dump(o: Any, **kwargs: Any) -> str: - # when python/mypy#1484 is solved, this can be `functools.partial` - return yaml.dump( - o, Dumper=Dumper, default_flow_style=False, indent=4, sort_keys=False, - **kwargs, - ) - def force_bytes(exc: Any) -> bytes: with contextlib.suppress(TypeError): @@ -52,18 +36,6 @@ def clean_path_on_failure(path: str) -> Generator[None, None, None]: raise -@contextlib.contextmanager -def tmpdir() -> Generator[str, None, None]: - """Contextmanager to create a temporary directory. It will be cleaned up - afterwards. - """ - tempdir = tempfile.mkdtemp() - try: - yield tempdir - finally: - rmtree(tempdir) - - def resource_bytesio(filename: str) -> IO[bytes]: return importlib.resources.open_binary('pre_commit.resources', filename) @@ -127,7 +99,7 @@ def cmd_output_b( _setdefault_kwargs(kwargs) try: - cmd = parse_shebang.normalize_cmd(cmd) + cmd = parse_shebang.normalize_cmd(cmd, env=kwargs.get('env')) except parse_shebang.ExecutableNotFoundError as e: returncode, stdout_b, stderr_b = e.to_output() else: @@ -254,10 +226,5 @@ def rmtree(path: str) -> None: shutil.rmtree(path, ignore_errors=False, onerror=handle_remove_readonly) -def parse_version(s: str) -> tuple[int, ...]: - """poor man's version comparison""" - return tuple(int(p) for p in s.split('.')) - - def win_exe(s: str) -> str: return s if sys.platform != 'win32' else f'{s}.exe' diff --git a/pre_commit/yaml.py b/pre_commit/yaml.py new file mode 100644 index 0000000..bdf4ec4 --- /dev/null +++ b/pre_commit/yaml.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import functools +from typing import Any + +import yaml + +Loader = getattr(yaml, 'CSafeLoader', yaml.SafeLoader) +yaml_load = functools.partial(yaml.load, Loader=Loader) +Dumper = getattr(yaml, 'CSafeDumper', yaml.SafeDumper) + + +def yaml_dump(o: Any, **kwargs: Any) -> str: + # when python/mypy#1484 is solved, this can be `functools.partial` + return yaml.dump( + o, Dumper=Dumper, default_flow_style=False, indent=4, sort_keys=False, + **kwargs, + ) |