From 2e8bfde15d76ad56da2d1bbd99294dafd4e4372f Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 4 Jan 2023 08:22:22 +0100 Subject: Merging upstream version 2.21.0. Signed-off-by: Daniel Baumann --- pre_commit/clientlib.py | 10 ++- pre_commit/commands/run.py | 12 ++- pre_commit/commands/sample_config.py | 4 - pre_commit/error_handler.py | 2 +- pre_commit/git.py | 26 +++--- pre_commit/languages/dotnet.py | 67 +++++++++++++--- pre_commit/languages/node.py | 2 +- pre_commit/languages/r.py | 5 +- pre_commit/languages/rust.py | 140 ++++++++++++++++++++++++--------- pre_commit/main.py | 8 ++ pre_commit/resources/ruby-build.tar.gz | Bin 72569 -> 74032 bytes pre_commit/staged_files_only.py | 2 +- pre_commit/util.py | 15 ++-- pre_commit/xargs.py | 2 +- 14 files changed, 206 insertions(+), 89 deletions(-) (limited to 'pre_commit') diff --git a/pre_commit/clientlib.py b/pre_commit/clientlib.py index 9b53e81..da6ca2b 100644 --- a/pre_commit/clientlib.py +++ b/pre_commit/clientlib.py @@ -298,6 +298,14 @@ CONFIG_HOOK_DICT = cfgv.Map( OptionalSensibleRegexAtHook('files', cfgv.check_string), OptionalSensibleRegexAtHook('exclude', cfgv.check_string), ) +LOCAL_HOOK_DICT = cfgv.Map( + 'Hook', 'id', + + *MANIFEST_HOOK_DICT.items, + + OptionalSensibleRegexAtHook('files', cfgv.check_string), + OptionalSensibleRegexAtHook('exclude', cfgv.check_string), +) CONFIG_REPO_DICT = cfgv.Map( 'Repository', 'repo', @@ -308,7 +316,7 @@ CONFIG_REPO_DICT = cfgv.Map( 'repo', cfgv.NotIn(LOCAL, META), ), cfgv.ConditionalRecurse( - 'hooks', cfgv.Array(MANIFEST_HOOK_DICT), + 'hooks', cfgv.Array(LOCAL_HOOK_DICT), 'repo', LOCAL, ), cfgv.ConditionalRecurse( diff --git a/pre_commit/commands/run.py b/pre_commit/commands/run.py index ad3d766..429e04c 100644 --- a/pre_commit/commands/run.py +++ b/pre_commit/commands/run.py @@ -263,7 +263,7 @@ def _all_filenames(args: argparse.Namespace) -> Collection[str]: def _get_diff() -> bytes: _, out, _ = cmd_output_b( - 'git', 'diff', '--no-ext-diff', '--ignore-submodules', retcode=None, + 'git', 'diff', '--no-ext-diff', '--ignore-submodules', check=False, ) return out @@ -318,7 +318,7 @@ def _has_unmerged_paths() -> bool: def _has_unstaged_config(config_file: str) -> bool: retcode, _, _ = cmd_output_b( 'git', 'diff', '--no-ext-diff', '--exit-code', config_file, - retcode=None, + check=False, ) # be explicit, other git errors don't mean it has an unstaged config. return retcode == 1 @@ -333,7 +333,7 @@ def run( stash = not args.all_files and not args.files # Check if we have unresolved merge conflict files and fail fast. - if _has_unmerged_paths(): + if stash and _has_unmerged_paths(): logger.error('Unmerged files. Resolve before committing.') return 1 if bool(args.from_ref) != bool(args.to_ref): @@ -420,7 +420,11 @@ def run( return 1 skips = _get_skips(environ) - to_install = [hook for hook in hooks if hook.id not in skips] + to_install = [ + hook + for hook in hooks + if hook.id not in skips and hook.alias not in skips + ] install_hook_envs(to_install, store) return _run_hooks(config, hooks, skips, args) diff --git a/pre_commit/commands/sample_config.py b/pre_commit/commands/sample_config.py index 82a1617..ce22f65 100644 --- a/pre_commit/commands/sample_config.py +++ b/pre_commit/commands/sample_config.py @@ -1,7 +1,3 @@ -# TODO: maybe `git ls-remote git://github.com/pre-commit/pre-commit-hooks` to -# determine the latest revision? This adds ~200ms from my tests (and is -# significantly faster than https:// or http://). For now, periodically -# manually updating the revision is fine. from __future__ import annotations SAMPLE_CONFIG = '''\ # See https://pre-commit.com for more information diff --git a/pre_commit/error_handler.py b/pre_commit/error_handler.py index 992f5cd..d740ee3 100644 --- a/pre_commit/error_handler.py +++ b/pre_commit/error_handler.py @@ -25,7 +25,7 @@ def _log_and_exit( error_msg = f'{msg}: {type(exc).__name__}: '.encode() + force_bytes(exc) output.write_line_b(error_msg) - _, git_version_b, _ = cmd_output_b('git', '--version', retcode=None) + _, git_version_b, _ = cmd_output_b('git', '--version', check=False) git_version = git_version_b.decode(errors='backslashreplace').rstrip() storedir = Store().directory diff --git a/pre_commit/git.py b/pre_commit/git.py index 35392b3..a76118f 100644 --- a/pre_commit/git.py +++ b/pre_commit/git.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging import os.path import sys -from typing import MutableMapping +from typing import Mapping from pre_commit.errors import FatalError from pre_commit.util import CalledProcessError @@ -24,9 +24,7 @@ def zsplit(s: str) -> list[str]: return [] -def no_git_env( - _env: MutableMapping[str, str] | None = None, -) -> dict[str, str]: +def no_git_env(_env: Mapping[str, str] | None = None) -> dict[str, str]: # Too many bugs dealing with environment variables and GIT: # https://github.com/pre-commit/pre-commit/issues/300 # In git 2.6.3 (maybe others), git exports GIT_WORK_TREE while running @@ -44,6 +42,8 @@ def no_git_env( 'GIT_EXEC_PATH', 'GIT_SSH', 'GIT_SSH_COMMAND', 'GIT_SSL_CAINFO', 'GIT_SSL_NO_VERIFY', 'GIT_CONFIG_COUNT', 'GIT_HTTP_PROXY_AUTHMETHOD', + 'GIT_ALLOW_PROTOCOL', + 'GIT_ASKPASS', } } @@ -150,18 +150,10 @@ def get_staged_files(cwd: str | None = None) -> list[str]: def intent_to_add_files() -> list[str]: _, stdout, _ = cmd_output( - 'git', 'status', '--ignore-submodules', '--porcelain', '-z', + 'git', 'diff', '--no-ext-diff', '--ignore-submodules', + '--diff-filter=A', '--name-only', '-z', ) - parts = list(reversed(zsplit(stdout))) - intent_to_add = [] - while parts: - line = parts.pop() - status, filename = line[:3], line[3:] - if status[0] in {'C', 'R'}: # renames / moves have an additional arg - parts.pop() - if status[1] == 'A': - intent_to_add.append(filename) - return intent_to_add + return zsplit(stdout) def get_all_files() -> list[str]: @@ -187,11 +179,11 @@ def head_rev(remote: str) -> str: def has_diff(*args: str, repo: str = '.') -> bool: cmd = ('git', 'diff', '--quiet', '--no-ext-diff', *args) - return cmd_output_b(*cmd, cwd=repo, retcode=None)[0] == 1 + return cmd_output_b(*cmd, cwd=repo, check=False)[0] == 1 def has_core_hookpaths_set() -> bool: - _, out, _ = cmd_output_b('git', 'config', 'core.hooksPath', retcode=None) + _, out, _ = cmd_output_b('git', 'config', 'core.hooksPath', check=False) return bool(out.strip()) diff --git a/pre_commit/languages/dotnet.py b/pre_commit/languages/dotnet.py index 3983c6f..e26b45c 100644 --- a/pre_commit/languages/dotnet.py +++ b/pre_commit/languages/dotnet.py @@ -2,6 +2,10 @@ from __future__ import annotations import contextlib import os.path +import re +import tempfile +import xml.etree.ElementTree +import zipfile from typing import Generator from typing import Sequence @@ -35,6 +39,22 @@ def in_env(prefix: Prefix) -> Generator[None, None, None]: yield +@contextlib.contextmanager +def _nuget_config_no_sources() -> Generator[str, None, None]: + with tempfile.TemporaryDirectory() as tmpdir: + nuget_config = os.path.join(tmpdir, 'nuget.config') + with open(nuget_config, 'w') as f: + f.write( + '' + '' + ' ' + ' ' + ' ' + '', + ) + yield nuget_config + + def install_environment( prefix: Prefix, version: str, @@ -57,21 +77,42 @@ def install_environment( ), ) - # Determine tool from the packaged file ..nupkg - build_outputs = os.listdir(os.path.join(prefix.prefix_dir, build_dir)) - for output in build_outputs: - tool_name = output.split('.')[0] + nupkg_dir = prefix.path(build_dir) + nupkgs = [x for x in os.listdir(nupkg_dir) if x.endswith('.nupkg')] + + if not nupkgs: + raise AssertionError('could not find any build outputs to install') + + for nupkg in nupkgs: + with zipfile.ZipFile(os.path.join(nupkg_dir, nupkg)) as f: + nuspec, = (x for x in f.namelist() if x.endswith('.nuspec')) + with f.open(nuspec) as spec: + tree = xml.etree.ElementTree.parse(spec) + + namespace = re.match(r'{.*}', tree.getroot().tag) + if not namespace: + raise AssertionError('could not parse namespace from nuspec') + + tool_id_element = tree.find(f'.//{namespace[0]}id') + if tool_id_element is None: + raise AssertionError('expected to find an "id" element') + + tool_id = tool_id_element.text + if not tool_id: + raise AssertionError('"id" element missing tool name') # Install to bin dir - helpers.run_setup_cmd( - prefix, - ( - 'dotnet', 'tool', 'install', - '--tool-path', os.path.join(envdir, BIN_DIR), - '--add-source', build_dir, - tool_name, - ), - ) + with _nuget_config_no_sources() as nuget_config: + helpers.run_setup_cmd( + prefix, + ( + 'dotnet', 'tool', 'install', + '--configfile', nuget_config, + '--tool-path', os.path.join(envdir, BIN_DIR), + '--add-source', build_dir, + tool_id, + ), + ) # Clean the git dir, ignoring the environment dir clean_cmd = ('git', 'clean', '-ffxd', '-e', f'{ENVIRONMENT_DIR}-*') diff --git a/pre_commit/languages/node.py b/pre_commit/languages/node.py index 39f3000..37a5b63 100644 --- a/pre_commit/languages/node.py +++ b/pre_commit/languages/node.py @@ -75,7 +75,7 @@ def in_env( def health_check(prefix: Prefix, language_version: str) -> str | None: with in_env(prefix, language_version): - retcode, _, _ = cmd_output_b('node', '--version', retcode=None) + retcode, _, _ = cmd_output_b('node', '--version', check=False) if retcode != 0: # pragma: win32 no cover return f'`node --version` returned {retcode}' else: diff --git a/pre_commit/languages/r.py b/pre_commit/languages/r.py index 40a001d..d281102 100644 --- a/pre_commit/languages/r.py +++ b/pre_commit/languages/r.py @@ -15,6 +15,7 @@ from pre_commit.languages import helpers from pre_commit.prefix import Prefix from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output_b +from pre_commit.util import win_exe ENVIRONMENT_DIR = 'renv' RSCRIPT_OPTS = ('--no-save', '--no-restore', '--no-site-file', '--no-environ') @@ -63,7 +64,7 @@ def _rscript_exec() -> str: if r_home is None: return 'Rscript' else: - return os.path.join(r_home, 'bin', 'Rscript') + return os.path.join(r_home, 'bin', win_exe('Rscript')) def _entry_validate(entry: Sequence[str]) -> None: @@ -158,7 +159,7 @@ def _inline_r_setup(code: str) -> str: only be configured via R options once R has started. These are set here. """ with_option = f"""\ - options(install.packages.compile.from.source = "never") + options(install.packages.compile.from.source = "never", pkgType = "binary") {code} """ return with_option diff --git a/pre_commit/languages/rust.py b/pre_commit/languages/rust.py index 01c3730..204f2aa 100644 --- a/pre_commit/languages/rust.py +++ b/pre_commit/languages/rust.py @@ -1,13 +1,17 @@ from __future__ import annotations import contextlib +import functools import os.path +import shutil +import sys +import tempfile +import urllib.request from typing import Generator from typing import Sequence -import toml - import pre_commit.constants as C +from pre_commit import parse_shebang from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var @@ -16,40 +20,105 @@ from pre_commit.languages import helpers from pre_commit.prefix import Prefix from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output_b +from pre_commit.util import make_executable +from pre_commit.util import win_exe ENVIRONMENT_DIR = 'rustenv' -get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check -def get_env_patch(target_dir: str) -> PatchesT: +@functools.lru_cache(maxsize=1) +def get_default_version() -> str: + # If rust is already installed, we can save a bunch of setup time by + # using the installed version. + # + # Just detecting the executable does not suffice, because if rustup is + # installed but no toolchain is available, then `cargo` exists but + # cannot be used without installing a toolchain first. + if cmd_output_b('cargo', '--version', check=False)[0] == 0: + return 'system' + else: + return C.DEFAULT + + +def _rust_toolchain(language_version: str) -> str: + """Transform the language version into a rust toolchain version.""" + if language_version == C.DEFAULT: + return 'stable' + else: + return language_version + + +def _envdir(prefix: Prefix, version: str) -> str: + directory = helpers.environment_dir(ENVIRONMENT_DIR, version) + return prefix.path(directory) + + +def get_env_patch(target_dir: str, version: str) -> PatchesT: return ( + ('CARGO_HOME', target_dir), ('PATH', (os.path.join(target_dir, 'bin'), os.pathsep, Var('PATH'))), + # Only set RUSTUP_TOOLCHAIN if we don't want use the system's default + # toolchain + *( + (('RUSTUP_TOOLCHAIN', _rust_toolchain(version)),) + if version != 'system' else () + ), ) @contextlib.contextmanager -def in_env(prefix: Prefix) -> Generator[None, None, None]: - target_dir = prefix.path( - helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), - ) - with envcontext(get_env_patch(target_dir)): +def in_env( + prefix: Prefix, + language_version: str, +) -> Generator[None, None, None]: + with envcontext( + get_env_patch(_envdir(prefix, language_version), language_version), + ): yield def _add_dependencies( - cargo_toml_path: str, + prefix: Prefix, additional_dependencies: set[str], ) -> None: - with open(cargo_toml_path, 'r+') as f: - cargo_toml = toml.load(f) - cargo_toml.setdefault('dependencies', {}) - for dep in additional_dependencies: - name, _, spec = dep.partition(':') - cargo_toml['dependencies'][name] = spec or '*' - f.seek(0) - toml.dump(cargo_toml, f) - f.truncate() + crates = [] + for dep in additional_dependencies: + name, _, spec = dep.partition(':') + crate = f'{name}@{spec or "*"}' + crates.append(crate) + + helpers.run_setup_cmd(prefix, ('cargo', 'add', *crates)) + + +def install_rust_with_toolchain(toolchain: str) -> None: + with tempfile.TemporaryDirectory() as rustup_dir: + with envcontext((('RUSTUP_HOME', rustup_dir),)): + # acquire `rustup` if not present + if parse_shebang.find_executable('rustup') is None: + # We did not detect rustup and need to download it first. + if sys.platform == 'win32': # pragma: win32 cover + url = 'https://win.rustup.rs/x86_64' + else: # pragma: win32 no cover + url = 'https://sh.rustup.rs' + + resp = urllib.request.urlopen(url) + + rustup_init = os.path.join(rustup_dir, win_exe('rustup-init')) + with open(rustup_init, 'wb') as f: + shutil.copyfileobj(resp, f) + make_executable(rustup_init) + + # install rustup into `$CARGO_HOME/bin` + cmd_output_b( + rustup_init, '-y', '--quiet', '--no-modify-path', + '--default-toolchain', 'none', + ) + + cmd_output_b( + 'rustup', 'toolchain', 'install', '--no-self-update', + toolchain, + ) def install_environment( @@ -57,10 +126,7 @@ def install_environment( version: str, additional_dependencies: Sequence[str], ) -> None: - helpers.assert_version_default('rust', version) - directory = prefix.path( - helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), - ) + directory = _envdir(prefix, version) # There are two cases where we might want to specify more dependencies: # as dependencies for the library being built, and as binary packages @@ -77,24 +143,28 @@ def install_environment( } lib_deps = set(additional_dependencies) - cli_deps - if len(lib_deps) > 0: - _add_dependencies(prefix.path('Cargo.toml'), lib_deps) - with clean_path_on_failure(directory): packages_to_install: set[tuple[str, ...]] = {('--path', '.')} for cli_dep in cli_deps: cli_dep = cli_dep[len('cli:'):] - package, _, version = cli_dep.partition(':') - if version != '': - packages_to_install.add((package, '--version', version)) + package, _, crate_version = cli_dep.partition(':') + if crate_version != '': + packages_to_install.add((package, '--version', crate_version)) else: packages_to_install.add((package,)) - for args in packages_to_install: - cmd_output_b( - 'cargo', 'install', '--bins', '--root', directory, *args, - cwd=prefix.prefix_dir, - ) + with in_env(prefix, version): + if version != 'system': + install_rust_with_toolchain(_rust_toolchain(version)) + + if len(lib_deps) > 0: + _add_dependencies(prefix, lib_deps) + + for args in packages_to_install: + cmd_output_b( + 'cargo', 'install', '--bins', '--root', directory, *args, + cwd=prefix.prefix_dir, + ) def run_hook( @@ -102,5 +172,5 @@ def run_hook( file_args: Sequence[str], color: bool, ) -> tuple[int, bytes]: - with in_env(hook.prefix): + with in_env(hook.prefix, hook.language_version): return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/main.py b/pre_commit/main.py index b4fa966..3915993 100644 --- a/pre_commit/main.py +++ b/pre_commit/main.py @@ -155,6 +155,10 @@ def _adjust_args_and_chdir(args: argparse.Namespace) -> None: args.config = os.path.abspath(args.config) if args.command in {'run', 'try-repo'}: args.files = [os.path.abspath(filename) for filename in args.files] + if args.commit_msg_filename is not None: + args.commit_msg_filename = os.path.abspath( + args.commit_msg_filename, + ) if args.command == 'try-repo' and os.path.exists(args.repo): args.repo = os.path.abspath(args.repo) @@ -164,6 +168,10 @@ def _adjust_args_and_chdir(args: argparse.Namespace) -> None: args.config = os.path.relpath(args.config) if args.command in {'run', 'try-repo'}: args.files = [os.path.relpath(filename) for filename in args.files] + if args.commit_msg_filename is not None: + args.commit_msg_filename = os.path.relpath( + args.commit_msg_filename, + ) if args.command == 'try-repo' and os.path.exists(args.repo): args.repo = os.path.relpath(args.repo) diff --git a/pre_commit/resources/ruby-build.tar.gz b/pre_commit/resources/ruby-build.tar.gz index 8edb3ca..35419f6 100644 Binary files a/pre_commit/resources/ruby-build.tar.gz and b/pre_commit/resources/ruby-build.tar.gz differ diff --git a/pre_commit/staged_files_only.py b/pre_commit/staged_files_only.py index 83d8a03..172fb20 100644 --- a/pre_commit/staged_files_only.py +++ b/pre_commit/staged_files_only.py @@ -52,7 +52,7 @@ def _unstaged_changes_cleared(patch_dir: str) -> Generator[None, None, None]: retcode, diff_stdout_binary, _ = cmd_output_b( 'git', 'diff-index', '--ignore-submodules', '--binary', '--exit-code', '--no-color', '--no-ext-diff', tree, '--', - retcode=None, + check=False, ) if retcode and diff_stdout_binary.strip(): patch_filename = f'patch{int(time.time())}-{os.getpid()}' diff --git a/pre_commit/util.py b/pre_commit/util.py index 8c296f4..b850768 100644 --- a/pre_commit/util.py +++ b/pre_commit/util.py @@ -83,14 +83,12 @@ class CalledProcessError(RuntimeError): self, returncode: int, cmd: tuple[str, ...], - expected_returncode: int, stdout: bytes, stderr: bytes | None, ) -> None: - super().__init__(returncode, cmd, expected_returncode, stdout, stderr) + super().__init__(returncode, cmd, stdout, stderr) self.returncode = returncode self.cmd = cmd - self.expected_returncode = expected_returncode self.stdout = stdout self.stderr = stderr @@ -104,7 +102,6 @@ class CalledProcessError(RuntimeError): return b''.join(( f'command: {self.cmd!r}\n'.encode(), f'return code: {self.returncode}\n'.encode(), - f'expected return code: {self.expected_returncode}\n'.encode(), b'stdout:', _indent_or_none(self.stdout), b'\n', b'stderr:', _indent_or_none(self.stderr), )) @@ -124,7 +121,7 @@ def _oserror_to_output(e: OSError) -> tuple[int, bytes, None]: def cmd_output_b( *cmd: str, - retcode: int | None = 0, + check: bool = True, **kwargs: Any, ) -> tuple[int, bytes, bytes | None]: _setdefault_kwargs(kwargs) @@ -142,8 +139,8 @@ def cmd_output_b( stdout_b, stderr_b = proc.communicate() returncode = proc.returncode - if retcode is not None and retcode != returncode: - raise CalledProcessError(returncode, cmd, retcode, stdout_b, stderr_b) + if check and returncode: + raise CalledProcessError(returncode, cmd, stdout_b, stderr_b) return returncode, stdout_b, stderr_b @@ -196,10 +193,10 @@ if os.name != 'nt': # pragma: win32 no cover def cmd_output_p( *cmd: str, - retcode: int | None = 0, + check: bool = True, **kwargs: Any, ) -> tuple[int, bytes, bytes | None]: - assert retcode is None + assert check is False assert kwargs['stderr'] == subprocess.STDOUT, kwargs['stderr'] _setdefault_kwargs(kwargs) diff --git a/pre_commit/xargs.py b/pre_commit/xargs.py index f2b3421..e3af90e 100644 --- a/pre_commit/xargs.py +++ b/pre_commit/xargs.py @@ -154,7 +154,7 @@ def xargs( run_cmd: tuple[str, ...], ) -> tuple[int, bytes, bytes | None]: return cmd_fn( - *run_cmd, retcode=None, stderr=subprocess.STDOUT, **kwargs, + *run_cmd, check=False, stderr=subprocess.STDOUT, **kwargs, ) threads = min(len(partitions), target_concurrency) -- cgit v1.2.3