diff options
65 files changed, 1119 insertions, 189 deletions
@@ -6,3 +6,4 @@ /.tox /dist /venv* +.vscode/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9cf739..80fa14b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.1.0 + rev: v3.3.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -12,25 +12,25 @@ repos: - id: requirements-txt-fixer - id: double-quote-string-fixer - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.3 + rev: 3.8.4 hooks: - id: flake8 - additional_dependencies: [flake8-typing-imports==1.6.0] + additional_dependencies: [flake8-typing-imports==1.10.0] - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v1.5.3 + rev: v1.5.4 hooks: - id: autopep8 - repo: https://github.com/pre-commit/pre-commit - rev: v2.6.0 + rev: v2.7.1 hooks: - id: validate_manifest - repo: https://github.com/asottile/pyupgrade - rev: v2.6.2 + rev: v2.7.3 hooks: - id: pyupgrade args: [--py36-plus] - repo: https://github.com/asottile/reorder_python_imports - rev: v2.3.0 + rev: v2.3.5 hooks: - id: reorder-python-imports args: [--py3-plus] @@ -40,11 +40,11 @@ repos: - id: add-trailing-comma args: [--py36-plus] - repo: https://github.com/asottile/setup-cfg-fmt - rev: v1.10.0 + rev: v1.15.1 hooks: - id: setup-cfg-fmt - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.782 + rev: v0.790 hooks: - id: mypy exclude: ^testing/resources/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a92a6b3..ff1013f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,72 @@ +2.8.2 - 2020-10-30 +================== + +### Fixes +- Fix installation of ruby hooks with `language_version: default` + - #1671 issue by @aerickson. + - #1672 PR by @asottile. + +2.8.1 - 2020-10-28 +================== + +### Fixes +- Allow default `language_version` of `system` when the homedir is `/` + - #1669 PR by @asottile. + +2.8.0 - 2020-10-28 +================== + +### Features +- Update `rbenv` / `ruby-build` + - #1612 issue by @tdeo. + - #1614 PR by @asottile. +- Update `sample-config` versions + - #1611 PR by @mcsitter. +- Add new language: `dotnet` + - #1598 by @rkm. +- Add `--negate` option to `language: pygrep` hooks + - #1643 PR by @MarcoGorelli. +- Add zipapp support + - #1616 PR by @asottile. +- Run pre-commit through https://pre-commit.ci + - #1662 PR by @asottile. +- Add new language: `coursier` (a jvm-based package manager) + - #1633 PR by @JosephMoniz. +- Exit with distinct codes: 1 (user error), 3 (unexpected error), 130 (^C) + - #1601 PR by @int3l. + +### Fixes +- Improve `healthy()` check for `language: node` + `language_version: system` + hooks when the system executable goes missing. + - pre-commit/action#45 issue by @KOliver94. + - #1589 issue by @asottile. + - #1590 PR by @asottile. +- Fix excess whitespace in error log traceback + - #1592 PR by @asottile. +- Fix posixlike shebang invocations with shim executables of the git hook + script on windows. + - #1593 issue by @Celeborn2BeAlive. + - #1595 PR by @Celeborn2BeAlive. +- Remove hard-coded `C:\PythonXX\python.exe` path on windows as it caused + confusion (and `virtualenv` can sometimes do better) + - #1599 PR by @asottile. +- Fix `language: ruby` hooks when `--format-executable` is present in a gemrc + - issue by `Rainbow Tux` (discord). + - #1603 PR by @asottile. +- Move `cygwin` / `win32` mismatch error earlier to catch msys2 mismatches + - #1605 issue by @danyeaw. + - #1606 PR by @asottile. +- Remove `-p` workaround for old `virtualenv` + - #1617 PR by @asottile. +- Fix `language: node` installations to not symlink outside of the environment + - pre-commit-ci/issues#2 issue by @DanielJSottile. + - #1667 PR by @asottile. +- Don't identify shim executables as valid `system` for defaulting + `language_version` for `language: node` / `language: ruby` + - #1658 issue by @adithyabsk. + - #1668 PR by @asottile. + + 2.7.1 - 2020-08-23 ================== @@ -1108,7 +1177,7 @@ that have helped us get this far! 0.18.1 - 2017-09-04 =================== - Only mention locking when waiting for a lock. -- Fix `IOError` during locking in timeout situtation on windows under python 2. +- Fix `IOError` during locking in timeout situation on windows under python 2. 0.18.0 - 2017-09-02 =================== @@ -1,6 +1,6 @@ [](https://dev.azure.com/asottile/asottile/_build/latest?definitionId=21&branchName=master) [](https://dev.azure.com/asottile/asottile/_build/latest?definitionId=21&branchName=master) -[](https://github.com/pre-commit/pre-commit) +[](https://results.pre-commit.ci/latest/github/pre-commit/pre-commit/master) ## pre-commit diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fb40010..e7256da 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,7 +13,6 @@ resources: ref: refs/tags/v2.0.0 jobs: -- template: job--pre-commit.yml@asottile - template: job--python-tox.yml@asottile parameters: toxenvs: [py37] @@ -36,6 +35,10 @@ jobs: - task: UseRubyVersion@0 - template: step--git-install.yml - bash: | + testing/get-coursier.sh + echo '##vso[task.prependpath]/tmp/coursier' + displayName: install coursier + - bash: | testing/get-swift.sh echo '##vso[task.prependpath]/tmp/swift/usr/bin' displayName: install swift @@ -46,6 +49,10 @@ jobs: pre_test: - task: UseRubyVersion@0 - bash: | + testing/get-coursier.sh + echo '##vso[task.prependpath]/tmp/coursier' + displayName: install coursier + - bash: | testing/get-swift.sh echo '##vso[task.prependpath]/tmp/swift/usr/bin' displayName: install swift diff --git a/pre_commit/clientlib.py b/pre_commit/clientlib.py index 8dfa947..87679bf 100644 --- a/pre_commit/clientlib.py +++ b/pre_commit/clientlib.py @@ -13,7 +13,7 @@ from identify.identify import ALL_TAGS import pre_commit.constants as C from pre_commit.color import add_color_option -from pre_commit.error_handler import FatalError +from pre_commit.errors import FatalError from pre_commit.languages.all import all_languages from pre_commit.logging_handler import logging_handler from pre_commit.util import parse_version diff --git a/pre_commit/commands/install_uninstall.py b/pre_commit/commands/install_uninstall.py index 85fa53c..684b598 100644 --- a/pre_commit/commands/install_uninstall.py +++ b/pre_commit/commands/install_uninstall.py @@ -55,7 +55,7 @@ def is_our_script(filename: str) -> bool: def shebang() -> str: if sys.platform == 'win32': - py = SYS_EXE + py, _ = os.path.splitext(SYS_EXE) else: exe_choices = [ f'python{sys.version_info[0]}.{sys.version_info[1]}', diff --git a/pre_commit/commands/run.py b/pre_commit/commands/run.py index 1f28c8c..0d335e2 100644 --- a/pre_commit/commands/run.py +++ b/pre_commit/commands/run.py @@ -11,6 +11,7 @@ from typing import Any from typing import Collection from typing import Dict from typing import List +from typing import MutableMapping from typing import Sequence from typing import Set from typing import Tuple @@ -28,7 +29,6 @@ from pre_commit.repository import install_hook_envs from pre_commit.staged_files_only import staged_files_only from pre_commit.store import Store from pre_commit.util import cmd_output_b -from pre_commit.util import EnvironT logger = logging.getLogger('pre_commit') @@ -116,7 +116,7 @@ class Classifier: return Classifier(filenames) -def _get_skips(environ: EnvironT) -> Set[str]: +def _get_skips(environ: MutableMapping[str, str]) -> Set[str]: skips = environ.get('SKIP', '') return {skip.strip() for skip in skips.split(',') if skip.strip()} @@ -258,7 +258,7 @@ def _run_hooks( config: Dict[str, Any], hooks: Sequence[Hook], args: argparse.Namespace, - environ: EnvironT, + environ: MutableMapping[str, str], ) -> int: """Actually run the hooks.""" skips = _get_skips(environ) @@ -315,7 +315,7 @@ def run( config_file: str, store: Store, args: argparse.Namespace, - environ: EnvironT = os.environ, + environ: MutableMapping[str, str] = os.environ, ) -> int: stash = not args.all_files and not args.files diff --git a/pre_commit/commands/sample_config.py b/pre_commit/commands/sample_config.py index d435faa..64617c3 100644 --- a/pre_commit/commands/sample_config.py +++ b/pre_commit/commands/sample_config.py @@ -7,7 +7,7 @@ SAMPLE_CONFIG = '''\ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v3.2.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer diff --git a/pre_commit/envcontext.py b/pre_commit/envcontext.py index 16d3d15..4ab0d8c 100644 --- a/pre_commit/envcontext.py +++ b/pre_commit/envcontext.py @@ -2,13 +2,12 @@ import contextlib import enum import os from typing import Generator +from typing import MutableMapping from typing import NamedTuple from typing import Optional from typing import Tuple from typing import Union -from pre_commit.util import EnvironT - class _Unset(enum.Enum): UNSET = 1 @@ -27,7 +26,7 @@ ValueT = Union[str, _Unset, SubstitutionT] PatchesT = Tuple[Tuple[str, ValueT], ...] -def format_env(parts: SubstitutionT, env: EnvironT) -> str: +def format_env(parts: SubstitutionT, env: MutableMapping[str, str]) -> str: return ''.join( env.get(part.name, part.default) if isinstance(part, Var) else part for part in parts @@ -37,7 +36,7 @@ def format_env(parts: SubstitutionT, env: EnvironT) -> str: @contextlib.contextmanager def envcontext( patch: PatchesT, - _env: Optional[EnvironT] = None, + _env: Optional[MutableMapping[str, str]] = None, ) -> Generator[None, None, None]: """In this context, `os.environ` is modified according to `patch`. @@ -50,7 +49,7 @@ def envcontext( replaced with the previous environment """ env = os.environ if _env is None else _env - before = env.copy() + before = dict(env) for k, v in patch: if v is UNSET: diff --git a/pre_commit/error_handler.py b/pre_commit/error_handler.py index 13d78cb..023dd35 100644 --- a/pre_commit/error_handler.py +++ b/pre_commit/error_handler.py @@ -7,15 +7,17 @@ from typing import Generator import pre_commit.constants as C from pre_commit import output +from pre_commit.errors import FatalError from pre_commit.store import Store from pre_commit.util import force_bytes -class FatalError(RuntimeError): - pass - - -def _log_and_exit(msg: str, exc: BaseException, formatted: str) -> None: +def _log_and_exit( + msg: str, + ret_code: int, + exc: BaseException, + formatted: str, +) -> None: error_msg = f'{msg}: {type(exc).__name__}: '.encode() + force_bytes(exc) output.write_line_b(error_msg) @@ -52,9 +54,9 @@ def _log_and_exit(msg: str, exc: BaseException, formatted: str) -> None: _log_line('```') _log_line() _log_line('```') - _log_line(formatted) + _log_line(formatted.rstrip()) _log_line('```') - raise SystemExit(1) + raise SystemExit(ret_code) @contextlib.contextmanager @@ -63,9 +65,9 @@ def error_handler() -> Generator[None, None, None]: yield except (Exception, KeyboardInterrupt) as e: if isinstance(e, FatalError): - msg = 'An error has occurred' + msg, ret_code = 'An error has occurred', 1 elif isinstance(e, KeyboardInterrupt): - msg = 'Interrupted (^C)' + msg, ret_code = 'Interrupted (^C)', 130 else: - msg = 'An unexpected error has occurred' - _log_and_exit(msg, e, traceback.format_exc()) + msg, ret_code = 'An unexpected error has occurred', 3 + _log_and_exit(msg, ret_code, e, traceback.format_exc()) diff --git a/pre_commit/errors.py b/pre_commit/errors.py new file mode 100644 index 0000000..f84d3f1 --- /dev/null +++ b/pre_commit/errors.py @@ -0,0 +1,2 @@ +class FatalError(RuntimeError): + pass diff --git a/pre_commit/git.py b/pre_commit/git.py index 576bef8..13ba664 100644 --- a/pre_commit/git.py +++ b/pre_commit/git.py @@ -3,12 +3,14 @@ import os.path import sys from typing import Dict from typing import List +from typing import MutableMapping from typing import Optional from typing import Set +from pre_commit.errors import FatalError +from pre_commit.util import CalledProcessError from pre_commit.util import cmd_output from pre_commit.util import cmd_output_b -from pre_commit.util import EnvironT logger = logging.getLogger(__name__) @@ -22,7 +24,9 @@ def zsplit(s: str) -> List[str]: return [] -def no_git_env(_env: Optional[EnvironT] = None) -> Dict[str, str]: +def no_git_env( + _env: Optional[MutableMapping[str, str]] = None, +) -> Dict[str, str]: # Too many bugs dealing with environment variables and GIT: # https://github.com/pre-commit/pre-commit/issues/300 # In git 2.6.3 (maybe others), git exports GIT_WORK_TREE while running @@ -43,7 +47,21 @@ def no_git_env(_env: Optional[EnvironT] = None) -> Dict[str, str]: def get_root() -> str: - return cmd_output('git', 'rev-parse', '--show-toplevel')[1].strip() + try: + root = cmd_output('git', 'rev-parse', '--show-toplevel')[1].strip() + except CalledProcessError: + raise FatalError( + 'git failed. Is it installed, and are you in a Git repository ' + 'directory?', + ) + else: + if root == '': # pragma: no cover (old git) + raise FatalError( + 'git toplevel unexpectedly empty! make sure you are not ' + 'inside the `.git` directory of your repository.', + ) + else: + return root def get_git_dir(git_root: str = '.') -> str: @@ -181,7 +199,7 @@ def check_for_cygwin_mismatch() -> None: """See https://github.com/pre-commit/pre-commit/issues/354""" if sys.platform in ('cygwin', 'win32'): # pragma: no cover (windows) is_cygwin_python = sys.platform == 'cygwin' - toplevel = cmd_output('git', 'rev-parse', '--show-toplevel')[1] + toplevel = get_root() is_cygwin_git = toplevel.startswith('/') if is_cygwin_python ^ is_cygwin_git: diff --git a/pre_commit/languages/all.py b/pre_commit/languages/all.py index 5609631..9c2e59d 100644 --- a/pre_commit/languages/all.py +++ b/pre_commit/languages/all.py @@ -6,8 +6,10 @@ from typing import Tuple from pre_commit.hook import Hook from pre_commit.languages import conda +from pre_commit.languages import coursier from pre_commit.languages import docker from pre_commit.languages import docker_image +from pre_commit.languages import dotnet from pre_commit.languages import fail from pre_commit.languages import golang from pre_commit.languages import node @@ -40,8 +42,10 @@ class Language(NamedTuple): languages = { # BEGIN GENERATED (testing/gen-languages-all) 'conda': Language(name='conda', ENVIRONMENT_DIR=conda.ENVIRONMENT_DIR, get_default_version=conda.get_default_version, healthy=conda.healthy, install_environment=conda.install_environment, run_hook=conda.run_hook), # noqa: E501 + 'coursier': Language(name='coursier', ENVIRONMENT_DIR=coursier.ENVIRONMENT_DIR, get_default_version=coursier.get_default_version, healthy=coursier.healthy, install_environment=coursier.install_environment, run_hook=coursier.run_hook), # noqa: E501 'docker': Language(name='docker', ENVIRONMENT_DIR=docker.ENVIRONMENT_DIR, get_default_version=docker.get_default_version, healthy=docker.healthy, install_environment=docker.install_environment, run_hook=docker.run_hook), # noqa: E501 'docker_image': Language(name='docker_image', ENVIRONMENT_DIR=docker_image.ENVIRONMENT_DIR, get_default_version=docker_image.get_default_version, healthy=docker_image.healthy, install_environment=docker_image.install_environment, run_hook=docker_image.run_hook), # noqa: E501 + 'dotnet': Language(name='dotnet', ENVIRONMENT_DIR=dotnet.ENVIRONMENT_DIR, get_default_version=dotnet.get_default_version, healthy=dotnet.healthy, install_environment=dotnet.install_environment, run_hook=dotnet.run_hook), # noqa: E501 'fail': Language(name='fail', ENVIRONMENT_DIR=fail.ENVIRONMENT_DIR, get_default_version=fail.get_default_version, healthy=fail.healthy, install_environment=fail.install_environment, run_hook=fail.run_hook), # noqa: E501 'golang': Language(name='golang', ENVIRONMENT_DIR=golang.ENVIRONMENT_DIR, get_default_version=golang.get_default_version, healthy=golang.healthy, install_environment=golang.install_environment, run_hook=golang.run_hook), # noqa: E501 'node': Language(name='node', ENVIRONMENT_DIR=node.ENVIRONMENT_DIR, get_default_version=node.get_default_version, healthy=node.healthy, install_environment=node.install_environment, run_hook=node.run_hook), # noqa: E501 diff --git a/pre_commit/languages/conda.py b/pre_commit/languages/conda.py index 071757a..d634e49 100644 --- a/pre_commit/languages/conda.py +++ b/pre_commit/languages/conda.py @@ -77,7 +77,7 @@ def run_hook( color: bool, ) -> Tuple[int, bytes]: # TODO: Some rare commands need to be run using `conda run` but mostly we - # can run them withot which is much quicker and produces a better + # can run them without which is much quicker and produces a better # output. # cmd = ('conda', 'run', '-p', env_dir) + hook.cmd with in_env(hook.prefix, hook.language_version): diff --git a/pre_commit/languages/coursier.py b/pre_commit/languages/coursier.py new file mode 100644 index 0000000..2841467 --- /dev/null +++ b/pre_commit/languages/coursier.py @@ -0,0 +1,71 @@ +import contextlib +import os +from typing import Generator +from typing import Sequence +from typing import Tuple + +from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT +from pre_commit.envcontext import Var +from pre_commit.hook import Hook +from pre_commit.languages import helpers +from pre_commit.prefix import Prefix +from pre_commit.util import clean_path_on_failure + +ENVIRONMENT_DIR = 'coursier' + +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy + + +def install_environment( + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], +) -> None: # pragma: win32 no cover + helpers.assert_version_default('coursier', version) + helpers.assert_no_additional_deps('coursier', additional_dependencies) + + envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version)) + channel = prefix.path('.pre-commit-channel') + with clean_path_on_failure(envdir): + for app_descriptor in os.listdir(channel): + _, app_file = os.path.split(app_descriptor) + app, _ = os.path.splitext(app_file) + helpers.run_setup_cmd( + prefix, + ( + 'cs', + 'install', + '--default-channels=false', + f'--channel={channel}', + app, + f'--dir={envdir}', + ), + ) + + +def get_env_patch(target_dir: str) -> PatchesT: # pragma: win32 no cover + return ( + ('PATH', (target_dir, os.pathsep, Var('PATH'))), + ) + + +@contextlib.contextmanager +def in_env( + prefix: Prefix, +) -> Generator[None, None, None]: # pragma: win32 no cover + target_dir = prefix.path( + helpers.environment_dir(ENVIRONMENT_DIR, get_default_version()), + ) + with envcontext(get_env_patch(target_dir)): + yield + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: # pragma: win32 no cover + with in_env(hook.prefix): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/docker.py b/pre_commit/languages/docker.py index 9c13119..9d30568 100644 --- a/pre_commit/languages/docker.py +++ b/pre_commit/languages/docker.py @@ -87,9 +87,8 @@ def run_hook( # automated cleanup of docker images. build_docker_image(hook.prefix, pull=False) - hook_cmd = hook.cmd - entry_exe, cmd_rest = hook.cmd[0], hook_cmd[1:] + entry_exe, *cmd_rest = hook.cmd entry_tag = ('--entrypoint', entry_exe, docker_tag(hook.prefix)) - cmd = docker_cmd() + entry_tag + cmd_rest + cmd = (*docker_cmd(), *entry_tag, *cmd_rest) return helpers.run_xargs(hook, cmd, file_args, color=color) diff --git a/pre_commit/languages/dotnet.py b/pre_commit/languages/dotnet.py new file mode 100644 index 0000000..a8abc86 --- /dev/null +++ b/pre_commit/languages/dotnet.py @@ -0,0 +1,90 @@ +import contextlib +import os.path +from typing import Generator +from typing import Sequence +from typing import Tuple + +import pre_commit.constants as C +from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT +from pre_commit.envcontext import Var +from pre_commit.hook import Hook +from pre_commit.languages import helpers +from pre_commit.prefix import Prefix +from pre_commit.util import clean_path_on_failure +from pre_commit.util import rmtree + +ENVIRONMENT_DIR = 'dotnetenv' +BIN_DIR = 'bin' + +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy + + +def get_env_patch(venv: str) -> PatchesT: + return ( + ('PATH', (os.path.join(venv, BIN_DIR), os.pathsep, Var('PATH'))), + ) + + +@contextlib.contextmanager +def in_env(prefix: Prefix) -> Generator[None, None, None]: + directory = helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT) + envdir = prefix.path(directory) + with envcontext(get_env_patch(envdir)): + yield + + +def install_environment( + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], +) -> None: + helpers.assert_version_default('dotnet', version) + helpers.assert_no_additional_deps('dotnet', additional_dependencies) + + envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version)) + with clean_path_on_failure(envdir): + build_dir = 'pre-commit-build' + + # Build & pack nupkg file + helpers.run_setup_cmd( + prefix, + ( + 'dotnet', 'pack', + '--configuration', 'Release', + '--output', build_dir, + ), + ) + + # Determine tool from the packaged file <tool_name>.<version>.nupkg + build_outputs = os.listdir(os.path.join(prefix.prefix_dir, build_dir)) + if len(build_outputs) != 1: + raise NotImplementedError( + f"Can't handle multiple build outputs. Got {build_outputs}", + ) + tool_name = build_outputs[0].split('.')[0] + + # Install to bin dir + helpers.run_setup_cmd( + prefix, + ( + 'dotnet', 'tool', 'install', + '--tool-path', os.path.join(envdir, BIN_DIR), + '--add-source', build_dir, + tool_name, + ), + ) + + # Cleanup build output + for d in ('bin', 'obj', build_dir): + rmtree(prefix.path(d)) + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: + with in_env(hook.prefix): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/helpers.py b/pre_commit/languages/helpers.py index 01c65ab..29138fd 100644 --- a/pre_commit/languages/helpers.py +++ b/pre_commit/languages/helpers.py @@ -1,6 +1,7 @@ import multiprocessing import os import random +import re from typing import Any from typing import List from typing import Optional @@ -10,6 +11,7 @@ from typing import Tuple from typing import TYPE_CHECKING import pre_commit.constants as C +from pre_commit import parse_shebang from pre_commit.hook import Hook from pre_commit.prefix import Prefix from pre_commit.util import cmd_output_b @@ -20,6 +22,31 @@ if TYPE_CHECKING: FIXED_RANDOM_SEED = 1542676187 +SHIMS_RE = re.compile(r'[/\\]shims[/\\]') + + +def exe_exists(exe: str) -> bool: + found = parse_shebang.find_executable(exe) + if found is None: # exe exists + return False + + homedir = os.path.expanduser('~') + try: + common: Optional[str] = os.path.commonpath((found, homedir)) + except ValueError: # on windows, different drives raises ValueError + common = None + + return ( + # it is not in a /shims/ directory + not SHIMS_RE.search(found) and + ( + # the homedir is / (docker, service user, etc.) + os.path.dirname(homedir) == homedir or + # the exe is not contained in the home directory + common != homedir + ) + ) + def run_setup_cmd(prefix: Prefix, cmd: Tuple[str, ...]) -> None: cmd_output_b(*cmd, cwd=prefix.prefix_dir) diff --git a/pre_commit/languages/node.py b/pre_commit/languages/node.py index d99e6f2..8dc4e8b 100644 --- a/pre_commit/languages/node.py +++ b/pre_commit/languages/node.py @@ -7,7 +7,6 @@ from typing import Sequence from typing import Tuple import pre_commit.constants as C -from pre_commit import parse_shebang from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import UNSET @@ -19,9 +18,9 @@ from pre_commit.prefix import Prefix from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output from pre_commit.util import cmd_output_b +from pre_commit.util import rmtree ENVIRONMENT_DIR = 'node_env' -healthy = helpers.basic_healthy @functools.lru_cache(maxsize=1) @@ -31,7 +30,7 @@ def get_default_version() -> str: return C.DEFAULT # if node is already installed, we can save a bunch of setup time by # using the installed version - elif all(parse_shebang.find_executable(exe) for exe in ('node', 'npm')): + elif all(helpers.exe_exists(exe) for exe in ('node', 'npm')): return 'system' else: return C.DEFAULT @@ -73,6 +72,12 @@ def in_env( yield +def healthy(prefix: Prefix, language_version: str) -> bool: + with in_env(prefix, language_version): + retcode, _, _ = cmd_output_b('node', '--version', retcode=None) + return retcode == 0 + + def install_environment( prefix: Prefix, version: str, additional_dependencies: Sequence[str], ) -> None: @@ -94,11 +99,23 @@ def install_environment( with in_env(prefix, version): # https://npm.community/t/npm-install-g-git-vs-git-clone-cd-npm-install-g/5449 # install as if we installed from git - helpers.run_setup_cmd(prefix, ('npm', 'install')) - helpers.run_setup_cmd( - prefix, - ('npm', 'install', '-g', '.', *additional_dependencies), + + local_install_cmd = ( + 'npm', 'install', '--dev', '--prod', + '--ignore-prepublish', '--no-progress', '--no-save', ) + helpers.run_setup_cmd(prefix, local_install_cmd) + + _, pkg, _ = cmd_output('npm', 'pack', cwd=prefix.prefix_dir) + pkg = prefix.path(pkg.strip()) + + install = ('npm', 'install', '-g', pkg, *additional_dependencies) + helpers.run_setup_cmd(prefix, install) + + # clean these up after installation + if prefix.exists('node_modules'): # pragma: win32 no cover + rmtree(prefix.path('node_modules')) + os.remove(pkg) def run_hook( diff --git a/pre_commit/languages/pygrep.py b/pre_commit/languages/pygrep.py index 40adba0..c80d679 100644 --- a/pre_commit/languages/pygrep.py +++ b/pre_commit/languages/pygrep.py @@ -1,6 +1,7 @@ import argparse import re import sys +from typing import NamedTuple from typing import Optional from typing import Pattern from typing import Sequence @@ -45,6 +46,46 @@ def _process_filename_at_once(pattern: Pattern[bytes], filename: str) -> int: return retv +def _process_filename_by_line_negated( + pattern: Pattern[bytes], + filename: str, +) -> int: + with open(filename, 'rb') as f: + for line in f: + if pattern.search(line): + return 0 + else: + output.write_line(filename) + return 1 + + +def _process_filename_at_once_negated( + pattern: Pattern[bytes], + filename: str, +) -> int: + with open(filename, 'rb') as f: + contents = f.read() + match = pattern.search(contents) + if match: + return 0 + else: + output.write_line(filename) + return 1 + + +class Choice(NamedTuple): + multiline: bool + negate: bool + + +FNS = { + Choice(multiline=True, negate=True): _process_filename_at_once_negated, + Choice(multiline=True, negate=False): _process_filename_at_once, + Choice(multiline=False, negate=True): _process_filename_by_line_negated, + Choice(multiline=False, negate=False): _process_filename_by_line, +} + + def run_hook( hook: Hook, file_args: Sequence[str], @@ -64,6 +105,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: ) parser.add_argument('-i', '--ignore-case', action='store_true') parser.add_argument('--multiline', action='store_true') + parser.add_argument('--negate', action='store_true') parser.add_argument('pattern', help='python regex pattern.') parser.add_argument('filenames', nargs='*') args = parser.parse_args(argv) @@ -75,11 +117,9 @@ def main(argv: Optional[Sequence[str]] = None) -> int: pattern = re.compile(args.pattern.encode(), flags) retv = 0 + process_fn = FNS[Choice(multiline=args.multiline, negate=args.negate)] for filename in args.filenames: - if args.multiline: - retv |= _process_filename_at_once(pattern, filename) - else: - retv |= _process_filename_by_line(pattern, filename) + retv |= process_fn(pattern, filename) return retv diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py index 7a68580..65f521c 100644 --- a/pre_commit/languages/python.py +++ b/pre_commit/languages/python.py @@ -114,11 +114,6 @@ def get_default_version() -> str: # pragma: no cover (platform dependent) if _find_by_py_launcher(exe): return exe - # Give a best-effort try for windows - default_folder_name = exe.replace('.', '') - if os.path.exists(fr'C:\{default_folder_name}\python.exe'): - return exe - # We tried! return C.DEFAULT @@ -137,13 +132,11 @@ def _sys_executable_matches(version: str) -> bool: return sys.version_info[:len(info)] == info -def norm_version(version: str) -> str: - if version == C.DEFAULT: - return os.path.realpath(sys.executable) - - # first see if our current executable is appropriate - if _sys_executable_matches(version): - return sys.executable +def norm_version(version: str) -> Optional[str]: + if version == C.DEFAULT: # use virtualenv's default + return None + elif _sys_executable_matches(version): # virtualenv defaults to our exe + return None if os.name == 'nt': # pragma: no cover (windows) version_exec = _find_by_py_launcher(version) @@ -155,12 +148,6 @@ def norm_version(version: str) -> str: if version_exec and version_exec != version: return version_exec - # If it is in the form pythonx.x search in the default - # place on windows - if version.startswith('python'): - default_folder_name = version.replace('.', '') - return fr'C:\{default_folder_name}\python.exe' - # Otherwise assume it is a path return os.path.expanduser(version) @@ -205,8 +192,10 @@ def install_environment( additional_dependencies: Sequence[str], ) -> None: envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version)) + venv_cmd = [sys.executable, '-mvirtualenv', envdir] python = norm_version(version) - venv_cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python) + if python is not None: + venv_cmd.extend(('-p', python)) install_cmd = ('python', '-mpip', 'install', '.', *additional_dependencies) with clean_path_on_failure(envdir): diff --git a/pre_commit/languages/ruby.py b/pre_commit/languages/ruby.py index 73b23cc..1a0f0c7 100644 --- a/pre_commit/languages/ruby.py +++ b/pre_commit/languages/ruby.py @@ -8,7 +8,6 @@ from typing import Sequence from typing import Tuple import pre_commit.constants as C -from pre_commit import parse_shebang from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import UNSET @@ -26,7 +25,7 @@ healthy = helpers.basic_healthy @functools.lru_cache(maxsize=1) def get_default_version() -> str: - if all(parse_shebang.find_executable(exe) for exe in ('ruby', 'gem')): + if all(helpers.exe_exists(exe) for exe in ('ruby', 'gem')): return 'system' else: return C.DEFAULT @@ -122,8 +121,8 @@ def install_environment( # Need to call this before installing so rbenv's directories # are set up helpers.run_setup_cmd(prefix, ('rbenv', 'init', '-')) - # XXX: this will *always* fail if `version == C.DEFAULT` - _install_ruby(prefix, version) + if version != C.DEFAULT: + _install_ruby(prefix, version) # Need to call this after installing to set up the shims helpers.run_setup_cmd(prefix, ('rbenv', 'rehash')) @@ -134,7 +133,8 @@ def install_environment( helpers.run_setup_cmd( prefix, ( - 'gem', 'install', '--no-document', + 'gem', 'install', + '--no-document', '--no-format-executable', *prefix.star('.gem'), *additional_dependencies, ), ) diff --git a/pre_commit/main.py b/pre_commit/main.py index 8647960..c1eb104 100644 --- a/pre_commit/main.py +++ b/pre_commit/main.py @@ -23,10 +23,8 @@ from pre_commit.commands.run import run from pre_commit.commands.sample_config import sample_config from pre_commit.commands.try_repo import try_repo from pre_commit.error_handler import error_handler -from pre_commit.error_handler import FatalError from pre_commit.logging_handler import logging_handler from pre_commit.store import Store -from pre_commit.util import CalledProcessError logger = logging.getLogger('pre_commit') @@ -146,21 +144,8 @@ def _adjust_args_and_chdir(args: argparse.Namespace) -> None: if args.command == 'try-repo' and os.path.exists(args.repo): args.repo = os.path.abspath(args.repo) - try: - toplevel = git.get_root() - except CalledProcessError: - raise FatalError( - 'git failed. Is it installed, and are you in a Git repository ' - 'directory?', - ) - else: - if toplevel == '': # pragma: no cover (old git) - raise FatalError( - 'git toplevel unexpectedly empty! make sure you are not ' - 'inside the `.git` directory of your repository.', - ) - else: - os.chdir(toplevel) + toplevel = git.get_root() + os.chdir(toplevel) args.config = os.path.relpath(args.config) if args.command in {'run', 'try-repo'}: @@ -339,11 +324,11 @@ def main(argv: Optional[Sequence[str]] = None) -> int: parser.parse_args(['--help']) with error_handler(), logging_handler(args.color): + git.check_for_cygwin_mismatch() + if args.command not in COMMANDS_NO_GIT: _adjust_args_and_chdir(args) - git.check_for_cygwin_mismatch() - store = Store() store.mark_config_used(args.config) diff --git a/pre_commit/make_archives.py b/pre_commit/make_archives.py index c31bcd7..d320b83 100644 --- a/pre_commit/make_archives.py +++ b/pre_commit/make_archives.py @@ -15,8 +15,8 @@ from pre_commit.util import tmpdir REPOS = ( - ('rbenv', 'git://github.com/rbenv/rbenv', 'a3fa9b7'), - ('ruby-build', 'git://github.com/rbenv/ruby-build', '1a902f3'), + ('rbenv', 'git://github.com/rbenv/rbenv', '0843745'), + ('ruby-build', 'git://github.com/rbenv/ruby-build', '258455e'), ( 'ruby-download', 'git://github.com/garnieretienne/rvm-download', diff --git a/pre_commit/resources/rbenv.tar.gz b/pre_commit/resources/rbenv.tar.gz Binary files differindex 5307b19..97ac469 100644 --- a/pre_commit/resources/rbenv.tar.gz +++ b/pre_commit/resources/rbenv.tar.gz diff --git a/pre_commit/resources/ruby-build.tar.gz b/pre_commit/resources/ruby-build.tar.gz Binary files differindex 4a69a09..4412ed4 100644 --- a/pre_commit/resources/ruby-build.tar.gz +++ b/pre_commit/resources/ruby-build.tar.gz diff --git a/pre_commit/util.py b/pre_commit/util.py index 0338b37..f4cf704 100644 --- a/pre_commit/util.py +++ b/pre_commit/util.py @@ -16,7 +16,6 @@ from typing import IO from typing import Optional from typing import Tuple from typing import Type -from typing import Union import yaml @@ -29,8 +28,6 @@ else: # pragma: no cover (<PY37) from importlib_resources import open_binary from importlib_resources import read_text -EnvironT = Union[Dict[str, str], 'os._Environ'] - Loader = getattr(yaml, 'CSafeLoader', yaml.SafeLoader) yaml_load = functools.partial(yaml.load, Loader=Loader) Dumper = getattr(yaml, 'CSafeDumper', yaml.SafeDumper) diff --git a/pre_commit/xargs.py b/pre_commit/xargs.py index 5235dc6..7538b54 100644 --- a/pre_commit/xargs.py +++ b/pre_commit/xargs.py @@ -9,6 +9,7 @@ from typing import Callable from typing import Generator from typing import Iterable from typing import List +from typing import MutableMapping from typing import Optional from typing import Sequence from typing import Tuple @@ -17,13 +18,12 @@ from typing import TypeVar from pre_commit import parse_shebang from pre_commit.util import cmd_output_b from pre_commit.util import cmd_output_p -from pre_commit.util import EnvironT TArg = TypeVar('TArg') TRet = TypeVar('TRet') -def _environ_size(_env: Optional[EnvironT] = None) -> int: +def _environ_size(_env: Optional[MutableMapping[str, str]] = None) -> int: environ = _env if _env is not None else getattr(os, 'environb', os.environ) size = 8 * len(environ) # number of pointers in `envp` for k, v in environ.items(): diff --git a/requirements-dev.txt b/requirements-dev.txt index d6a13dc..56afd41 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,6 @@ covdefaults coverage +distlib pytest pytest-env +re-assert @@ -1,6 +1,6 @@ [metadata] name = pre_commit -version = 2.7.1 +version = 2.8.2 description = A framework for managing and maintaining multi-language pre-commit hooks. long_description = file: README.md long_description_content_type = text/markdown @@ -16,6 +16,7 @@ classifiers = Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy diff --git a/testing/gen-languages-all b/testing/gen-languages-all index 2bff7be..d9b01bd 100755 --- a/testing/gen-languages-all +++ b/testing/gen-languages-all @@ -2,8 +2,9 @@ import sys LANGUAGES = [ - 'conda', 'docker', 'docker_image', 'fail', 'golang', 'node', 'perl', - 'pygrep', 'python', 'ruby', 'rust', 'script', 'swift', 'system', + 'conda', 'coursier', 'docker', 'dotnet', 'docker_image', 'fail', 'golang', + 'node', 'perl', 'pygrep', 'python', 'ruby', 'rust', 'script', 'swift', + 'system', ] FIELDS = [ 'ENVIRONMENT_DIR', 'get_default_version', 'healthy', 'install_environment', diff --git a/testing/get-coursier.ps1 b/testing/get-coursier.ps1 new file mode 100755 index 0000000..42e5635 --- /dev/null +++ b/testing/get-coursier.ps1 @@ -0,0 +1,11 @@ +$wc = New-Object System.Net.WebClient + +$coursier_url = "https://github.com/coursier/coursier/releases/download/v2.0.5/cs-x86_64-pc-win32.exe" +$coursier_dest = "C:\coursier\cs.exe" +$coursier_hash ="d63d497f7805261e1cd657b8aaa626f6b8f7264cdb68219b2e6be9dd882033a9" + +New-Item -Path "C:\" -Name "coursier" -ItemType "directory" +$wc.DownloadFile($coursier_url, $coursier_dest) +if ((Get-FileHash $coursier_dest -Algorithm SHA256).Hash -ne $coursier_hash) { + throw "Invalid coursier file" +} diff --git a/testing/get-coursier.sh b/testing/get-coursier.sh new file mode 100755 index 0000000..760c6c1 --- /dev/null +++ b/testing/get-coursier.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# This is a script used in CI to install coursier +set -euxo pipefail + +COURSIER_URL="https://github.com/coursier/coursier/releases/download/v2.0.0/cs-x86_64-pc-linux" +COURSIER_HASH="e2e838b75bc71b16bcb77ce951ad65660c89bda7957c79a0628ec7146d35122f" +ARTIFACT="/tmp/coursier/cs" + +mkdir -p /tmp/coursier +rm -f "$ARTIFACT" +curl --location --silent --output "$ARTIFACT" "$COURSIER_URL" +echo "$COURSIER_HASH $ARTIFACT" | sha256sum --check +chmod ugo+x /tmp/coursier/cs diff --git a/testing/resources/coursier_hooks_repo/.pre-commit-channel/echo-java.json b/testing/resources/coursier_hooks_repo/.pre-commit-channel/echo-java.json new file mode 100644 index 0000000..37f401e --- /dev/null +++ b/testing/resources/coursier_hooks_repo/.pre-commit-channel/echo-java.json @@ -0,0 +1,8 @@ +{ + "repositories": [ + "central" + ], + "dependencies": [ + "io.get-coursier:echo:latest.stable" + ] +} diff --git a/testing/resources/coursier_hooks_repo/.pre-commit-hooks.yaml b/testing/resources/coursier_hooks_repo/.pre-commit-hooks.yaml new file mode 100644 index 0000000..d4a143b --- /dev/null +++ b/testing/resources/coursier_hooks_repo/.pre-commit-hooks.yaml @@ -0,0 +1,5 @@ +- id: echo-java + name: echo-java + description: echo from java + entry: echo-java + language: coursier diff --git a/testing/resources/dotnet_hooks_csproj_repo/.gitignore b/testing/resources/dotnet_hooks_csproj_repo/.gitignore new file mode 100644 index 0000000..edcd28f --- /dev/null +++ b/testing/resources/dotnet_hooks_csproj_repo/.gitignore @@ -0,0 +1,3 @@ +bin/ +obj/ +nupkg/ diff --git a/testing/resources/dotnet_hooks_csproj_repo/.pre-commit-hooks.yaml b/testing/resources/dotnet_hooks_csproj_repo/.pre-commit-hooks.yaml new file mode 100644 index 0000000..d005a74 --- /dev/null +++ b/testing/resources/dotnet_hooks_csproj_repo/.pre-commit-hooks.yaml @@ -0,0 +1,5 @@ +- id: dotnet example hook + name: dotnet example hook + entry: testeroni + language: dotnet + files: '' diff --git a/testing/resources/dotnet_hooks_csproj_repo/Program.cs b/testing/resources/dotnet_hooks_csproj_repo/Program.cs new file mode 100644 index 0000000..1456e8e --- /dev/null +++ b/testing/resources/dotnet_hooks_csproj_repo/Program.cs @@ -0,0 +1,12 @@ +using System; + +namespace dotnet_hooks_repo +{ + class Program + { + static void Main(string[] args) + { + Console.WriteLine("Hello from dotnet!"); + } + } +} diff --git a/testing/resources/dotnet_hooks_csproj_repo/dotnet_hooks_csproj_repo.csproj b/testing/resources/dotnet_hooks_csproj_repo/dotnet_hooks_csproj_repo.csproj new file mode 100644 index 0000000..d2e556a --- /dev/null +++ b/testing/resources/dotnet_hooks_csproj_repo/dotnet_hooks_csproj_repo.csproj @@ -0,0 +1,9 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <OutputType>Exe</OutputType> + <TargetFramework>netcoreapp3.1</TargetFramework> + <PackAsTool>true</PackAsTool> + <ToolCommandName>testeroni</ToolCommandName> + <PackageOutputPath>./nupkg</PackageOutputPath> + </PropertyGroup> +</Project> diff --git a/testing/resources/dotnet_hooks_sln_repo/.gitignore b/testing/resources/dotnet_hooks_sln_repo/.gitignore new file mode 100644 index 0000000..edcd28f --- /dev/null +++ b/testing/resources/dotnet_hooks_sln_repo/.gitignore @@ -0,0 +1,3 @@ +bin/ +obj/ +nupkg/ diff --git a/testing/resources/dotnet_hooks_sln_repo/.pre-commit-hooks.yaml b/testing/resources/dotnet_hooks_sln_repo/.pre-commit-hooks.yaml new file mode 100644 index 0000000..d005a74 --- /dev/null +++ b/testing/resources/dotnet_hooks_sln_repo/.pre-commit-hooks.yaml @@ -0,0 +1,5 @@ +- id: dotnet example hook + name: dotnet example hook + entry: testeroni + language: dotnet + files: '' diff --git a/testing/resources/dotnet_hooks_sln_repo/Program.cs b/testing/resources/dotnet_hooks_sln_repo/Program.cs new file mode 100644 index 0000000..04ad4e0 --- /dev/null +++ b/testing/resources/dotnet_hooks_sln_repo/Program.cs @@ -0,0 +1,12 @@ +using System;
+
+namespace dotnet_hooks_sln_repo
+{
+ class Program
+ {
+ static void Main(string[] args)
+ {
+ Console.WriteLine("Hello from dotnet!");
+ }
+ }
+}
diff --git a/testing/resources/dotnet_hooks_sln_repo/dotnet_hooks_sln_repo.csproj b/testing/resources/dotnet_hooks_sln_repo/dotnet_hooks_sln_repo.csproj new file mode 100644 index 0000000..e372964 --- /dev/null +++ b/testing/resources/dotnet_hooks_sln_repo/dotnet_hooks_sln_repo.csproj @@ -0,0 +1,9 @@ +<Project Sdk="Microsoft.NET.Sdk">
+ <PropertyGroup>
+ <OutputType>Exe</OutputType>
+ <TargetFramework>netcoreapp3.1</TargetFramework>
+ <PackAsTool>true</PackAsTool>
+ <ToolCommandName>testeroni</ToolCommandName>
+ <PackageOutputPath>./nupkg</PackageOutputPath>
+ </PropertyGroup>
+</Project>
diff --git a/testing/resources/dotnet_hooks_sln_repo/dotnet_hooks_sln_repo.sln b/testing/resources/dotnet_hooks_sln_repo/dotnet_hooks_sln_repo.sln new file mode 100644 index 0000000..87d2afb --- /dev/null +++ b/testing/resources/dotnet_hooks_sln_repo/dotnet_hooks_sln_repo.sln @@ -0,0 +1,34 @@ +
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.26124.0
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "dotnet_hooks_sln_repo", "dotnet_hooks_sln_repo.csproj", "{6568CFDB-6F6F-45A9-932C-8C7DAABC8E56}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Any CPU = Debug|Any CPU
+ Debug|x64 = Debug|x64
+ Debug|x86 = Debug|x86
+ Release|Any CPU = Release|Any CPU
+ Release|x64 = Release|x64
+ Release|x86 = Release|x86
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {6568CFDB-6F6F-45A9-932C-8C7DAABC8E56}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {6568CFDB-6F6F-45A9-932C-8C7DAABC8E56}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {6568CFDB-6F6F-45A9-932C-8C7DAABC8E56}.Debug|x64.ActiveCfg = Debug|Any CPU
+ {6568CFDB-6F6F-45A9-932C-8C7DAABC8E56}.Debug|x64.Build.0 = Debug|Any CPU
+ {6568CFDB-6F6F-45A9-932C-8C7DAABC8E56}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {6568CFDB-6F6F-45A9-932C-8C7DAABC8E56}.Debug|x86.Build.0 = Debug|Any CPU
+ {6568CFDB-6F6F-45A9-932C-8C7DAABC8E56}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {6568CFDB-6F6F-45A9-932C-8C7DAABC8E56}.Release|Any CPU.Build.0 = Release|Any CPU
+ {6568CFDB-6F6F-45A9-932C-8C7DAABC8E56}.Release|x64.ActiveCfg = Release|Any CPU
+ {6568CFDB-6F6F-45A9-932C-8C7DAABC8E56}.Release|x64.Build.0 = Release|Any CPU
+ {6568CFDB-6F6F-45A9-932C-8C7DAABC8E56}.Release|x86.ActiveCfg = Release|Any CPU
+ {6568CFDB-6F6F-45A9-932C-8C7DAABC8E56}.Release|x86.Build.0 = Release|Any CPU
+ EndGlobalSection
+EndGlobal
diff --git a/testing/util.py b/testing/util.py index f556a8d..18cd734 100644 --- a/testing/util.py +++ b/testing/util.py @@ -40,6 +40,10 @@ def cmd_output_mocked_pre_commit_home( return ret, out.replace('\r\n', '\n'), None +skipif_cant_run_coursier = pytest.mark.skipif( + os.name == 'nt' or parse_shebang.find_executable('cs') is None, + reason="coursier isn't installed or can't be found", +) skipif_cant_run_docker = pytest.mark.skipif( os.name == 'nt' or not docker_is_running(), reason="Docker isn't running or can't be accessed", diff --git a/testing/zipapp/Dockerfile b/testing/zipapp/Dockerfile new file mode 100644 index 0000000..e21d5fe --- /dev/null +++ b/testing/zipapp/Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:bionic +RUN : \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + python3 \ + python3-distutils \ + python3-venv \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ENV LANG=C.UTF-8 PATH=/venv/bin:$PATH +RUN : \ + && python3.6 -mvenv /venv \ + && pip install --no-cache-dir pip setuptools wheel no-manylinux --upgrade diff --git a/testing/zipapp/entry b/testing/zipapp/entry new file mode 100755 index 0000000..f0a345e --- /dev/null +++ b/testing/zipapp/entry @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +import os.path +import shutil +import stat +import sys +import tempfile +import zipfile + +from pre_commit.file_lock import lock + +CACHE_DIR = os.path.expanduser('~/.cache/pre-commit-zipapp') + + +def _make_executable(filename: str) -> None: + os.chmod(filename, os.stat(filename).st_mode | stat.S_IXUSR) + + +def _ensure_cache(zipf: zipfile.ZipFile, cache_key: str) -> str: + os.makedirs(CACHE_DIR, exist_ok=True) + + cache_dest = os.path.join(CACHE_DIR, cache_key) + lock_filename = os.path.join(CACHE_DIR, f'{cache_key}.lock') + + if os.path.exists(cache_dest): + return cache_dest + + with lock(lock_filename, blocked_cb=lambda: None): + # another process may have completed this work + if os.path.exists(cache_dest): + return cache_dest + + tmpdir = tempfile.mkdtemp(prefix=os.path.join(CACHE_DIR, '')) + try: + zipf.extractall(tmpdir) + # zip doesn't maintain permissions + _make_executable(os.path.join(tmpdir, 'python')) + _make_executable(os.path.join(tmpdir, 'python.exe')) + os.rename(tmpdir, cache_dest) + except BaseException: + shutil.rmtree(tmpdir) + raise + + return cache_dest + + +def main() -> int: + with zipfile.ZipFile(os.path.dirname(__file__)) as zipf: + with zipf.open('CACHE_KEY') as f: + cache_key = f.read().decode().strip() + + cache_dest = _ensure_cache(zipf, cache_key) + + if sys.platform != 'win32': + exe = os.path.join(cache_dest, 'python') + else: + exe = os.path.join(cache_dest, 'python.exe') + + cmd = (exe, '-mpre_commit', *sys.argv[1:]) + if sys.platform == 'win32': # https://bugs.python.org/issue19124 + import subprocess + + if sys.version_info < (3, 7): # https://bugs.python.org/issue25942 + return subprocess.Popen(cmd).wait() + else: + return subprocess.call(cmd) + else: + os.execvp(cmd[0], cmd) + + +if __name__ == '__main__': + exit(main()) diff --git a/testing/zipapp/make b/testing/zipapp/make new file mode 100755 index 0000000..a644946 --- /dev/null +++ b/testing/zipapp/make @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +import argparse +import base64 +import hashlib +import importlib.resources +import io +import os.path +import shutil +import subprocess +import tempfile +import zipapp +import zipfile + +HERE = os.path.dirname(os.path.realpath(__file__)) +IMG = 'make-pre-commit-zipapp' + + +def _msg(s: str) -> None: + print(f'\033[7m{s}\033[m') + + +def _exit_if_retv(*cmd: str) -> None: + if subprocess.call(cmd): + raise SystemExit(1) + + +def _check_no_shared_objects(wheeldir: str) -> None: + for zip_filename in os.listdir(wheeldir): + with zipfile.ZipFile(os.path.join(wheeldir, zip_filename)) as zipf: + for filename in zipf.namelist(): + if filename.endswith('.so') or '.so.' in filename: + raise AssertionError(zip_filename, filename) + + +def _add_shim(dest: str) -> None: + shim = os.path.join(HERE, 'python') + shutil.copy(shim, dest) + + bio = io.BytesIO() + with zipfile.ZipFile(bio, 'w') as zipf: + zipf.write(shim, arcname='__main__.py') + + with open(os.path.join(dest, 'python.exe'), 'wb') as f: + f.write(importlib.resources.read_binary('distlib', 't32.exe')) + f.write(b'#!py.exe -3\n') + f.write(bio.getvalue()) + + +def _write_cache_key(version: str, wheeldir: str, dest: str) -> None: + cache_hash = hashlib.sha256(f'{version}\n'.encode()) + for filename in sorted(os.listdir(wheeldir)): + cache_hash.update(f'{filename}\n'.encode()) + with open(os.path.join(HERE, 'python'), 'rb') as f: + cache_hash.update(f.read()) + with open(os.path.join(dest, 'CACHE_KEY'), 'wb') as f: + f.write(base64.urlsafe_b64encode(cache_hash.digest()).rstrip(b'=')) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument('version') + args = parser.parse_args() + + with tempfile.TemporaryDirectory() as tmpdir: + wheeldir = os.path.join(tmpdir, 'wheels') + os.mkdir(wheeldir) + + _msg('building podman image...') + _exit_if_retv('podman', 'build', '-q', '-t', IMG, HERE) + + _msg('populating wheels...') + _exit_if_retv( + 'podman', 'run', '--rm', '--volume', f'{wheeldir}:/wheels:rw', IMG, + 'pip', 'wheel', f'pre_commit=={args.version}', + '--wheel-dir', '/wheels', + ) + + _msg('validating wheels...') + _check_no_shared_objects(wheeldir) + + _msg('adding __main__.py...') + mainfile = os.path.join(tmpdir, '__main__.py') + shutil.copy(os.path.join(HERE, 'entry'), mainfile) + + _msg('adding shim...') + _add_shim(tmpdir) + + _msg('copying file_lock.py...') + file_lock_py = os.path.join(HERE, '../../pre_commit/file_lock.py') + file_lock_py_dest = os.path.join(tmpdir, 'pre_commit/file_lock.py') + os.makedirs(os.path.dirname(file_lock_py_dest)) + shutil.copy(file_lock_py, file_lock_py_dest) + + _msg('writing CACHE_KEY...') + _write_cache_key(args.version, wheeldir, tmpdir) + + filename = f'pre-commit-{args.version}.pyz' + _msg(f'writing {filename}...') + shebang = '/usr/bin/env python3' + zipapp.create_archive(tmpdir, filename, interpreter=shebang) + + return 0 + + +if __name__ == '__main__': + exit(main()) diff --git a/testing/zipapp/python b/testing/zipapp/python new file mode 100755 index 0000000..97c5928 --- /dev/null +++ b/testing/zipapp/python @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""A shim executable to put dependencies on sys.path""" +import argparse +import os.path +import runpy +import sys + +# an exe-zipapp will have a __file__ of shim.exe/__main__.py +EXE = __file__ if os.path.isfile(__file__) else os.path.dirname(__file__) +EXE = os.path.realpath(EXE) +HERE = os.path.dirname(EXE) +WHEELDIR = os.path.join(HERE, 'wheels') +SITE_DIRS = frozenset(('dist-packages', 'site-packages')) + + +def main() -> int: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('-m') + args, rest = parser.parse_known_args() + + if args.m: + # try and remove site-packages from sys.path so our packages win + sys.path[:] = [ + p for p in sys.path + if os.path.split(p)[1] not in SITE_DIRS + ] + for wheel in sorted(os.listdir(WHEELDIR)): + sys.path.append(os.path.join(WHEELDIR, wheel)) + if args.m == 'pre_commit' or args.m.startswith('pre_commit.'): + sys.executable = EXE + sys.argv[1:] = rest + runpy.run_module(args.m, run_name='__main__', alter_sys=True) + return 0 + else: + cmd = (sys.executable, *sys.argv[1:]) + if sys.platform == 'win32': # https://bugs.python.org/issue19124 + import subprocess + + if sys.version_info < (3, 7): # https://bugs.python.org/issue25942 + return subprocess.Popen(cmd).wait() + else: + return subprocess.call(cmd) + else: + os.execvp(cmd[0], cmd) + + +if __name__ == '__main__': + exit(main()) diff --git a/tests/commands/install_uninstall_test.py b/tests/commands/install_uninstall_test.py index 5809a3f..7a4b906 100644 --- a/tests/commands/install_uninstall_test.py +++ b/tests/commands/install_uninstall_test.py @@ -3,6 +3,8 @@ import re import sys from unittest import mock +import re_assert + import pre_commit.constants as C from pre_commit import git from pre_commit.commands import install_uninstall @@ -54,8 +56,13 @@ def patch_sys_exe(exe): def test_shebang_windows(): + with patch_platform('win32'), patch_sys_exe('python'): + assert shebang() == '#!/usr/bin/env python' + + +def test_shebang_windows_drop_ext(): with patch_platform('win32'), patch_sys_exe('python.exe'): - assert shebang() == '#!/usr/bin/env python.exe' + assert shebang() == '#!/usr/bin/env python' def test_shebang_posix_not_on_path(): @@ -143,7 +150,7 @@ FILES_CHANGED = ( ) -NORMAL_PRE_COMMIT_RUN = re.compile( +NORMAL_PRE_COMMIT_RUN = re_assert.Matches( fr'^\[INFO\] Initializing environment for .+\.\n' fr'Bash hook\.+Passed\n' fr'\[master [a-f0-9]{{7}}\] commit!\n' @@ -159,7 +166,7 @@ def test_install_pre_commit_and_run(tempdir_factory, store): ret, output = _get_commit_output(tempdir_factory) assert ret == 0 - assert NORMAL_PRE_COMMIT_RUN.match(output) + NORMAL_PRE_COMMIT_RUN.assert_matches(output) def test_install_pre_commit_and_run_custom_path(tempdir_factory, store): @@ -171,7 +178,7 @@ def test_install_pre_commit_and_run_custom_path(tempdir_factory, store): ret, output = _get_commit_output(tempdir_factory) assert ret == 0 - assert NORMAL_PRE_COMMIT_RUN.match(output) + NORMAL_PRE_COMMIT_RUN.assert_matches(output) def test_install_in_submodule_and_run(tempdir_factory, store): @@ -185,7 +192,7 @@ def test_install_in_submodule_and_run(tempdir_factory, store): assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 ret, output = _get_commit_output(tempdir_factory) assert ret == 0 - assert NORMAL_PRE_COMMIT_RUN.match(output) + NORMAL_PRE_COMMIT_RUN.assert_matches(output) def test_install_in_worktree_and_run(tempdir_factory, store): @@ -198,7 +205,7 @@ def test_install_in_worktree_and_run(tempdir_factory, store): assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 ret, output = _get_commit_output(tempdir_factory) assert ret == 0 - assert NORMAL_PRE_COMMIT_RUN.match(output) + NORMAL_PRE_COMMIT_RUN.assert_matches(output) def test_commit_am(tempdir_factory, store): @@ -243,7 +250,7 @@ def test_install_idempotent(tempdir_factory, store): ret, output = _get_commit_output(tempdir_factory) assert ret == 0 - assert NORMAL_PRE_COMMIT_RUN.match(output) + NORMAL_PRE_COMMIT_RUN.assert_matches(output) def _path_without_us(): @@ -297,7 +304,7 @@ def test_environment_not_sourced(tempdir_factory, store): ) -FAILING_PRE_COMMIT_RUN = re.compile( +FAILING_PRE_COMMIT_RUN = re_assert.Matches( r'^\[INFO\] Initializing environment for .+\.\n' r'Failing hook\.+Failed\n' r'- hook id: failing_hook\n' @@ -316,10 +323,10 @@ def test_failing_hooks_returns_nonzero(tempdir_factory, store): ret, output = _get_commit_output(tempdir_factory) assert ret == 1 - assert FAILING_PRE_COMMIT_RUN.match(output) + FAILING_PRE_COMMIT_RUN.assert_matches(output) -EXISTING_COMMIT_RUN = re.compile( +EXISTING_COMMIT_RUN = re_assert.Matches( fr'^legacy hook\n' fr'\[master [a-f0-9]{{7}}\] commit!\n' fr'{FILES_CHANGED}' @@ -342,7 +349,7 @@ def test_install_existing_hooks_no_overwrite(tempdir_factory, store): # Make sure we installed the "old" hook correctly ret, output = _get_commit_output(tempdir_factory, touch_file='baz') assert ret == 0 - assert EXISTING_COMMIT_RUN.match(output) + EXISTING_COMMIT_RUN.assert_matches(output) # Now install pre-commit (no-overwrite) assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 @@ -351,7 +358,7 @@ def test_install_existing_hooks_no_overwrite(tempdir_factory, store): ret, output = _get_commit_output(tempdir_factory) assert ret == 0 assert output.startswith('legacy hook\n') - assert NORMAL_PRE_COMMIT_RUN.match(output[len('legacy hook\n'):]) + NORMAL_PRE_COMMIT_RUN.assert_matches(output[len('legacy hook\n'):]) def test_legacy_overwriting_legacy_hook(tempdir_factory, store): @@ -377,10 +384,10 @@ def test_install_existing_hook_no_overwrite_idempotent(tempdir_factory, store): ret, output = _get_commit_output(tempdir_factory) assert ret == 0 assert output.startswith('legacy hook\n') - assert NORMAL_PRE_COMMIT_RUN.match(output[len('legacy hook\n'):]) + NORMAL_PRE_COMMIT_RUN.assert_matches(output[len('legacy hook\n'):]) -FAIL_OLD_HOOK = re.compile( +FAIL_OLD_HOOK = re_assert.Matches( r'fail!\n' r'\[INFO\] Initializing environment for .+\.\n' r'Bash hook\.+Passed\n', @@ -401,7 +408,7 @@ def test_failing_existing_hook_returns_1(tempdir_factory, store): # We should get a failure from the legacy hook ret, output = _get_commit_output(tempdir_factory) assert ret == 1 - assert FAIL_OLD_HOOK.match(output) + FAIL_OLD_HOOK.assert_matches(output) def test_install_overwrite_no_existing_hooks(tempdir_factory, store): @@ -413,7 +420,7 @@ def test_install_overwrite_no_existing_hooks(tempdir_factory, store): ret, output = _get_commit_output(tempdir_factory) assert ret == 0 - assert NORMAL_PRE_COMMIT_RUN.match(output) + NORMAL_PRE_COMMIT_RUN.assert_matches(output) def test_install_overwrite(tempdir_factory, store): @@ -426,7 +433,7 @@ def test_install_overwrite(tempdir_factory, store): ret, output = _get_commit_output(tempdir_factory) assert ret == 0 - assert NORMAL_PRE_COMMIT_RUN.match(output) + NORMAL_PRE_COMMIT_RUN.assert_matches(output) def test_uninstall_restores_legacy_hooks(tempdir_factory, store): @@ -441,7 +448,7 @@ def test_uninstall_restores_legacy_hooks(tempdir_factory, store): # Make sure we installed the "old" hook correctly ret, output = _get_commit_output(tempdir_factory, touch_file='baz') assert ret == 0 - assert EXISTING_COMMIT_RUN.match(output) + EXISTING_COMMIT_RUN.assert_matches(output) def test_replace_old_commit_script(tempdir_factory, store): @@ -463,7 +470,7 @@ def test_replace_old_commit_script(tempdir_factory, store): ret, output = _get_commit_output(tempdir_factory) assert ret == 0 - assert NORMAL_PRE_COMMIT_RUN.match(output) + NORMAL_PRE_COMMIT_RUN.assert_matches(output) def test_uninstall_doesnt_remove_not_our_hooks(in_git_dir): @@ -476,7 +483,7 @@ def test_uninstall_doesnt_remove_not_our_hooks(in_git_dir): assert pre_commit.exists() -PRE_INSTALLED = re.compile( +PRE_INSTALLED = re_assert.Matches( fr'Bash hook\.+Passed\n' fr'\[master [a-f0-9]{{7}}\] commit!\n' fr'{FILES_CHANGED}' @@ -493,7 +500,7 @@ def test_installs_hooks_with_hooks_True(tempdir_factory, store): ) assert ret == 0 - assert PRE_INSTALLED.match(output) + PRE_INSTALLED.assert_matches(output) def test_install_hooks_command(tempdir_factory, store): @@ -506,7 +513,7 @@ def test_install_hooks_command(tempdir_factory, store): ) assert ret == 0 - assert PRE_INSTALLED.match(output) + PRE_INSTALLED.assert_matches(output) def test_installed_from_venv(tempdir_factory, store): @@ -533,7 +540,7 @@ def test_installed_from_venv(tempdir_factory, store): }, ) assert ret == 0 - assert NORMAL_PRE_COMMIT_RUN.match(output) + NORMAL_PRE_COMMIT_RUN.assert_matches(output) def _get_push_output(tempdir_factory, remote='origin', opts=()): @@ -880,7 +887,7 @@ def test_prepare_commit_msg_legacy( def test_pre_merge_commit_integration(tempdir_factory, store): - expected = re.compile( + output_pattern = re_assert.Matches( r'^\[INFO\] Initializing environment for .+\n' r'Bash hook\.+Passed\n' r"Merge made by the 'recursive' strategy.\n" @@ -902,7 +909,7 @@ def test_pre_merge_commit_integration(tempdir_factory, store): tempdir_factory=tempdir_factory, ) assert ret == 0 - assert expected.match(output) + output_pattern.assert_matches(output) def test_install_disallow_missing_config(tempdir_factory, store): diff --git a/tests/commands/run_test.py b/tests/commands/run_test.py index 2461ed5..00b4712 100644 --- a/tests/commands/run_test.py +++ b/tests/commands/run_test.py @@ -2,6 +2,7 @@ import os.path import shlex import sys import time +from typing import MutableMapping from unittest import mock import pytest @@ -18,7 +19,6 @@ from pre_commit.commands.run import Classifier from pre_commit.commands.run import filter_by_include_exclude from pre_commit.commands.run import run from pre_commit.util import cmd_output -from pre_commit.util import EnvironT from pre_commit.util import make_executable from testing.auto_namedtuple import auto_namedtuple from testing.fixtures import add_config_to_repo @@ -482,7 +482,7 @@ def test_all_push_options_ok(cap_out, store, repo_with_passing_hook): def test_checkout_type(cap_out, store, repo_with_passing_hook): args = run_opts(from_ref='', to_ref='', checkout_type='1') - environ: EnvironT = {} + environ: MutableMapping[str, str] = {} ret, printed = _do_run( cap_out, store, repo_with_passing_hook, args, environ, ) @@ -1032,7 +1032,7 @@ def test_skipped_without_any_setup_for_post_checkout(in_git_dir, store): def test_pre_commit_env_variable_set(cap_out, store, repo_with_passing_hook): args = run_opts() - environ: EnvironT = {} + environ: MutableMapping[str, str] = {} ret, printed = _do_run( cap_out, store, repo_with_passing_hook, args, environ, ) diff --git a/tests/commands/sample_config_test.py b/tests/commands/sample_config_test.py index 11c0876..8e3a904 100644 --- a/tests/commands/sample_config_test.py +++ b/tests/commands/sample_config_test.py @@ -10,7 +10,7 @@ def test_sample_config(capsys): # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v3.2.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer diff --git a/tests/commands/try_repo_test.py b/tests/commands/try_repo_test.py index d3ec3fd..a157d16 100644 --- a/tests/commands/try_repo_test.py +++ b/tests/commands/try_repo_test.py @@ -3,6 +3,8 @@ import re import time from unittest import mock +import re_assert + from pre_commit import git from pre_commit.commands.try_repo import try_repo from pre_commit.util import cmd_output @@ -43,7 +45,7 @@ def test_try_repo_repo_only(cap_out, tempdir_factory): _run_try_repo(tempdir_factory, verbose=True) start, config, rest = _get_out(cap_out) assert start == '' - assert re.match( + config_pattern = re_assert.Matches( '^repos:\n' '- repo: .+\n' ' rev: .+\n' @@ -51,8 +53,8 @@ def test_try_repo_repo_only(cap_out, tempdir_factory): ' - id: bash_hook\n' ' - id: bash_hook2\n' ' - id: bash_hook3\n$', - config, ) + config_pattern.assert_matches(config) assert rest == '''\ Bash hook............................................(no files to check)Skipped - hook id: bash_hook @@ -71,14 +73,14 @@ def test_try_repo_with_specific_hook(cap_out, tempdir_factory): _run_try_repo(tempdir_factory, hook='bash_hook', verbose=True) start, config, rest = _get_out(cap_out) assert start == '' - assert re.match( + config_pattern = re_assert.Matches( '^repos:\n' '- repo: .+\n' ' rev: .+\n' ' hooks:\n' ' - id: bash_hook\n$', - config, ) + config_pattern.assert_matches(config) assert rest == '''\ Bash hook............................................(no files to check)Skipped - hook id: bash_hook @@ -128,14 +130,14 @@ def test_try_repo_uncommitted_changes(cap_out, tempdir_factory): start, config, rest = _get_out(cap_out) assert start == '[WARNING] Creating temporary repo with uncommitted changes...\n' # noqa: E501 - assert re.match( + config_pattern = re_assert.Matches( '^repos:\n' '- repo: .+shadow-repo\n' ' rev: .+\n' ' hooks:\n' ' - id: bash_hook\n$', - config, ) + config_pattern.assert_matches(config) assert rest == 'modified name!...........................................................Passed\n' # noqa: E501 diff --git a/tests/error_handler_test.py b/tests/error_handler_test.py index d066e57..6b0bb86 100644 --- a/tests/error_handler_test.py +++ b/tests/error_handler_test.py @@ -1,12 +1,13 @@ import os.path -import re import stat import sys from unittest import mock import pytest +import re_assert from pre_commit import error_handler +from pre_commit.errors import FatalError from pre_commit.store import Store from pre_commit.util import CalledProcessError from testing.util import cmd_output_mocked_pre_commit_home @@ -26,27 +27,28 @@ def test_error_handler_no_exception(mocked_log_and_exit): def test_error_handler_fatal_error(mocked_log_and_exit): - exc = error_handler.FatalError('just a test') + exc = FatalError('just a test') with error_handler.error_handler(): raise exc mocked_log_and_exit.assert_called_once_with( 'An error has occurred', + 1, exc, # Tested below mock.ANY, ) - assert re.match( + pattern = re_assert.Matches( r'Traceback \(most recent call last\):\n' r' File ".+pre_commit.error_handler.py", line \d+, in error_handler\n' r' yield\n' r' File ".+tests.error_handler_test.py", line \d+, ' r'in test_error_handler_fatal_error\n' r' raise exc\n' - r'(pre_commit\.error_handler\.)?FatalError: just a test\n', - mocked_log_and_exit.call_args[0][2], + r'(pre_commit\.errors\.)?FatalError: just a test\n', ) + pattern.assert_matches(mocked_log_and_exit.call_args[0][3]) def test_error_handler_uncaught_error(mocked_log_and_exit): @@ -56,11 +58,12 @@ def test_error_handler_uncaught_error(mocked_log_and_exit): mocked_log_and_exit.assert_called_once_with( 'An unexpected error has occurred', + 3, exc, # Tested below mock.ANY, ) - assert re.match( + pattern = re_assert.Matches( r'Traceback \(most recent call last\):\n' r' File ".+pre_commit.error_handler.py", line \d+, in error_handler\n' r' yield\n' @@ -68,8 +71,8 @@ def test_error_handler_uncaught_error(mocked_log_and_exit): r'in test_error_handler_uncaught_error\n' r' raise exc\n' r'ValueError: another test\n', - mocked_log_and_exit.call_args[0][2], ) + pattern.assert_matches(mocked_log_and_exit.call_args[0][3]) def test_error_handler_keyboardinterrupt(mocked_log_and_exit): @@ -79,11 +82,12 @@ def test_error_handler_keyboardinterrupt(mocked_log_and_exit): mocked_log_and_exit.assert_called_once_with( 'Interrupted (^C)', + 130, exc, # Tested below mock.ANY, ) - assert re.match( + pattern = re_assert.Matches( r'Traceback \(most recent call last\):\n' r' File ".+pre_commit.error_handler.py", line \d+, in error_handler\n' r' yield\n' @@ -91,15 +95,20 @@ def test_error_handler_keyboardinterrupt(mocked_log_and_exit): r'in test_error_handler_keyboardinterrupt\n' r' raise exc\n' r'KeyboardInterrupt\n', - mocked_log_and_exit.call_args[0][2], ) + pattern.assert_matches(mocked_log_and_exit.call_args[0][3]) def test_log_and_exit(cap_out, mock_store_dir): - with pytest.raises(SystemExit): - error_handler._log_and_exit( - 'msg', error_handler.FatalError('hai'), "I'm a stacktrace", - ) + tb = ( + 'Traceback (most recent call last):\n' + ' File "<stdin>", line 2, in <module>\n' + 'pre_commit.errors.FatalError: hai\n' + ) + + with pytest.raises(SystemExit) as excinfo: + error_handler._log_and_exit('msg', 1, FatalError('hai'), tb) + assert excinfo.value.code == 1 printed = cap_out.get() log_file = os.path.join(mock_store_dir, 'pre-commit.log') @@ -108,7 +117,7 @@ def test_log_and_exit(cap_out, mock_store_dir): assert os.path.exists(log_file) with open(log_file) as f: logged = f.read() - expected = ( + pattern = re_assert.Matches( r'^### version information\n' r'\n' r'```\n' @@ -127,10 +136,12 @@ def test_log_and_exit(cap_out, mock_store_dir): r'```\n' r'\n' r'```\n' - r"I'm a stacktrace\n" - r'```\n' + r'Traceback \(most recent call last\):\n' + r' File "<stdin>", line 2, in <module>\n' + r'pre_commit\.errors\.FatalError: hai\n' + r'```\n', ) - assert re.match(expected, logged) + pattern.assert_matches(logged) def test_error_handler_non_ascii_exception(mock_store_dir): @@ -163,7 +174,7 @@ def test_error_handler_no_tty(tempdir_factory): 'from pre_commit.error_handler import error_handler\n' 'with error_handler():\n' ' raise ValueError("\\u2603")\n', - retcode=1, + retcode=3, tempdir_factory=tempdir_factory, pre_commit_home=pre_commit_home, ) diff --git a/tests/languages/dotnet_test.py b/tests/languages/dotnet_test.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tests/languages/dotnet_test.py diff --git a/tests/languages/helpers_test.py b/tests/languages/helpers_test.py index fa493cc..669cd33 100644 --- a/tests/languages/helpers_test.py +++ b/tests/languages/helpers_test.py @@ -1,17 +1,66 @@ import multiprocessing -import os +import os.path import sys from unittest import mock import pytest import pre_commit.constants as C +from pre_commit import parse_shebang from pre_commit.languages import helpers from pre_commit.prefix import Prefix from pre_commit.util import CalledProcessError from testing.auto_namedtuple import auto_namedtuple +@pytest.fixture +def find_exe_mck(): + with mock.patch.object(parse_shebang, 'find_executable') as mck: + yield mck + + +@pytest.fixture +def homedir_mck(): + def fake_expanduser(pth): + assert pth == '~' + return os.path.normpath('/home/me') + + with mock.patch.object(os.path, 'expanduser', fake_expanduser): + yield + + +def test_exe_exists_does_not_exist(find_exe_mck, homedir_mck): + find_exe_mck.return_value = None + assert helpers.exe_exists('ruby') is False + + +def test_exe_exists_exists(find_exe_mck, homedir_mck): + find_exe_mck.return_value = os.path.normpath('/usr/bin/ruby') + assert helpers.exe_exists('ruby') is True + + +def test_exe_exists_false_if_shim(find_exe_mck, homedir_mck): + find_exe_mck.return_value = os.path.normpath('/foo/shims/ruby') + assert helpers.exe_exists('ruby') is False + + +def test_exe_exists_false_if_homedir(find_exe_mck, homedir_mck): + find_exe_mck.return_value = os.path.normpath('/home/me/somedir/ruby') + assert helpers.exe_exists('ruby') is False + + +def test_exe_exists_commonpath_raises_ValueError(find_exe_mck, homedir_mck): + find_exe_mck.return_value = os.path.normpath('/usr/bin/ruby') + with mock.patch.object(os.path, 'commonpath', side_effect=ValueError): + assert helpers.exe_exists('ruby') is True + + +def test_exe_exists_true_when_homedir_is_slash(find_exe_mck): + find_exe_mck.return_value = os.path.normpath('/usr/bin/ruby') + with mock.patch.object(os.path, 'expanduser', return_value=os.sep): + assert helpers.exe_exists('ruby') is True + + def test_basic_get_default_version(): assert helpers.basic_get_default_version() == C.DEFAULT diff --git a/tests/languages/node_test.py b/tests/languages/node_test.py index fd30046..8e52268 100644 --- a/tests/languages/node_test.py +++ b/tests/languages/node_test.py @@ -1,14 +1,21 @@ +import json +import os +import shutil import sys from unittest import mock import pytest import pre_commit.constants as C +from pre_commit import envcontext from pre_commit import parse_shebang -from pre_commit.languages.node import get_default_version +from pre_commit.languages import node +from pre_commit.prefix import Prefix +from pre_commit.util import cmd_output +from testing.util import xfailif_windows -ACTUAL_GET_DEFAULT_VERSION = get_default_version.__wrapped__ +ACTUAL_GET_DEFAULT_VERSION = node.get_default_version.__wrapped__ @pytest.fixture @@ -45,3 +52,57 @@ def test_uses_default_when_node_and_npm_are_not_available(find_exe_mck): def test_sets_default_on_windows(find_exe_mck): find_exe_mck.return_value = '/path/to/exe' assert ACTUAL_GET_DEFAULT_VERSION() == C.DEFAULT + + +@xfailif_windows # pragma: win32 no cover +def test_healthy_system_node(tmpdir): + tmpdir.join('package.json').write('{"name": "t", "version": "1.0.0"}') + + prefix = Prefix(str(tmpdir)) + node.install_environment(prefix, 'system', ()) + assert node.healthy(prefix, 'system') + + +@xfailif_windows # pragma: win32 no cover +def test_unhealthy_if_system_node_goes_missing(tmpdir): + bin_dir = tmpdir.join('bin').ensure_dir() + node_bin = bin_dir.join('node') + node_bin.mksymlinkto(shutil.which('node')) + + prefix_dir = tmpdir.join('prefix').ensure_dir() + prefix_dir.join('package.json').write('{"name": "t", "version": "1.0.0"}') + + path = ('PATH', (str(bin_dir), os.pathsep, envcontext.Var('PATH'))) + with envcontext.envcontext((path,)): + prefix = Prefix(str(prefix_dir)) + node.install_environment(prefix, 'system', ()) + assert node.healthy(prefix, 'system') + + node_bin.remove() + assert not node.healthy(prefix, 'system') + + +@xfailif_windows # pragma: win32 no cover +def test_installs_without_links_outside_env(tmpdir): + tmpdir.join('bin/main.js').ensure().write( + '#!/usr/bin/env node\n' + '_ = require("lodash"); console.log("success!")\n', + ) + tmpdir.join('package.json').write( + json.dumps({ + 'name': 'foo', + 'version': '0.0.1', + 'bin': {'foo': './bin/main.js'}, + 'dependencies': {'lodash': '*'}, + }), + ) + + prefix = Prefix(str(tmpdir)) + node.install_environment(prefix, 'system', ()) + assert node.healthy(prefix, 'system') + + # this directory shouldn't exist, make sure we succeed without it existing + cmd_output('rm', '-rf', str(tmpdir.join('node_modules'))) + + with node.in_env(prefix, 'system'): + assert cmd_output('foo')[1] == 'success!\n' diff --git a/tests/languages/pygrep_test.py b/tests/languages/pygrep_test.py index cabea22..d8bacc4 100644 --- a/tests/languages/pygrep_test.py +++ b/tests/languages/pygrep_test.py @@ -8,6 +8,9 @@ def some_files(tmpdir): tmpdir.join('f1').write_binary(b'foo\nbar\n') tmpdir.join('f2').write_binary(b'[INFO] hi\n') tmpdir.join('f3').write_binary(b"with'quotes\n") + tmpdir.join('f4').write_binary(b'foo\npattern\nbar\n') + tmpdir.join('f5').write_binary(b'[INFO] hi\npattern\nbar') + tmpdir.join('f6').write_binary(b"pattern\nbarwith'foo\n") with tmpdir.as_cwd(): yield @@ -23,42 +26,99 @@ def some_files(tmpdir): ("h'q", 1, "f3:1:with'quotes\n"), ), ) -def test_main(some_files, cap_out, pattern, expected_retcode, expected_out): +def test_main(cap_out, pattern, expected_retcode, expected_out): ret = pygrep.main((pattern, 'f1', 'f2', 'f3')) out = cap_out.get() assert ret == expected_retcode assert out == expected_out -def test_ignore_case(some_files, cap_out): +@pytest.mark.usefixtures('some_files') +def test_negate_by_line_no_match(cap_out): + ret = pygrep.main(('pattern\nbar', 'f4', 'f5', 'f6', '--negate')) + out = cap_out.get() + assert ret == 1 + assert out == 'f4\nf5\nf6\n' + + +@pytest.mark.usefixtures('some_files') +def test_negate_by_line_two_match(cap_out): + ret = pygrep.main(('foo', 'f4', 'f5', 'f6', '--negate')) + out = cap_out.get() + assert ret == 1 + assert out == 'f5\n' + + +@pytest.mark.usefixtures('some_files') +def test_negate_by_line_all_match(cap_out): + ret = pygrep.main(('pattern', 'f4', 'f5', 'f6', '--negate')) + out = cap_out.get() + assert ret == 0 + assert out == '' + + +@pytest.mark.usefixtures('some_files') +def test_negate_by_file_no_match(cap_out): + ret = pygrep.main(('baz', 'f4', 'f5', 'f6', '--negate', '--multiline')) + out = cap_out.get() + assert ret == 1 + assert out == 'f4\nf5\nf6\n' + + +@pytest.mark.usefixtures('some_files') +def test_negate_by_file_one_match(cap_out): + ret = pygrep.main( + ('foo\npattern', 'f4', 'f5', 'f6', '--negate', '--multiline'), + ) + out = cap_out.get() + assert ret == 1 + assert out == 'f5\nf6\n' + + +@pytest.mark.usefixtures('some_files') +def test_negate_by_file_all_match(cap_out): + ret = pygrep.main( + ('pattern\nbar', 'f4', 'f5', 'f6', '--negate', '--multiline'), + ) + out = cap_out.get() + assert ret == 0 + assert out == '' + + +@pytest.mark.usefixtures('some_files') +def test_ignore_case(cap_out): ret = pygrep.main(('--ignore-case', 'info', 'f1', 'f2', 'f3')) out = cap_out.get() assert ret == 1 assert out == 'f2:1:[INFO] hi\n' -def test_multiline(some_files, cap_out): +@pytest.mark.usefixtures('some_files') +def test_multiline(cap_out): ret = pygrep.main(('--multiline', r'foo\nbar', 'f1', 'f2', 'f3')) out = cap_out.get() assert ret == 1 assert out == 'f1:1:foo\nbar\n' -def test_multiline_line_number(some_files, cap_out): +@pytest.mark.usefixtures('some_files') +def test_multiline_line_number(cap_out): ret = pygrep.main(('--multiline', r'ar', 'f1', 'f2', 'f3')) out = cap_out.get() assert ret == 1 assert out == 'f1:2:bar\n' -def test_multiline_dotall_flag_is_enabled(some_files, cap_out): +@pytest.mark.usefixtures('some_files') +def test_multiline_dotall_flag_is_enabled(cap_out): ret = pygrep.main(('--multiline', r'o.*bar', 'f1', 'f2', 'f3')) out = cap_out.get() assert ret == 1 assert out == 'f1:1:foo\nbar\n' -def test_multiline_multiline_flag_is_enabled(some_files, cap_out): +@pytest.mark.usefixtures('some_files') +def test_multiline_multiline_flag_is_enabled(cap_out): ret = pygrep.main(('--multiline', r'foo$.*bar', 'f1', 'f2', 'f3')) out = cap_out.get() assert ret == 1 diff --git a/tests/languages/python_test.py b/tests/languages/python_test.py index 29c5a9b..cfe1483 100644 --- a/tests/languages/python_test.py +++ b/tests/languages/python_test.py @@ -36,13 +36,14 @@ def test_norm_version_expanduser(): def test_norm_version_of_default_is_sys_executable(): - assert python.norm_version('default') == os.path.realpath(sys.executable) + assert python.norm_version('default') is None @pytest.mark.parametrize('v', ('python3.6', 'python3', 'python')) def test_sys_executable_matches(v): with mock.patch.object(sys, 'version_info', (3, 6, 7)): assert python._sys_executable_matches(v) + assert python.norm_version(v) is None @pytest.mark.parametrize('v', ('notpython', 'python3.x')) diff --git a/tests/languages/ruby_test.py b/tests/languages/ruby_test.py index 853bb73..6c0c9e5 100644 --- a/tests/languages/ruby_test.py +++ b/tests/languages/ruby_test.py @@ -30,23 +30,45 @@ def test_uses_system_if_both_gem_and_ruby_are_available(find_exe_mck): assert ACTUAL_GET_DEFAULT_VERSION() == 'system' +@pytest.fixture +def fake_gem_prefix(tmpdir): + gemspec = '''\ +Gem::Specification.new do |s| + s.name = 'pre_commit_dummy_package' + s.version = '0.0.0' + s.summary = 'dummy gem for pre-commit hooks' + s.authors = ['Anthony Sottile'] +end +''' + tmpdir.join('dummy_gem.gemspec').write(gemspec) + yield Prefix(tmpdir) + + +@xfailif_windows # pragma: win32 no cover +def test_install_ruby_system(fake_gem_prefix): + ruby.install_environment(fake_gem_prefix, 'system', ()) + + # Should be able to activate and use rbenv install + with ruby.in_env(fake_gem_prefix, 'system'): + _, out, _ = cmd_output('gem', 'list') + assert 'pre_commit_dummy_package' in out + + @xfailif_windows # pragma: win32 no cover -def test_install_rbenv(tempdir_factory): - prefix = Prefix(tempdir_factory.get()) - ruby._install_rbenv(prefix, C.DEFAULT) +def test_install_ruby_default(fake_gem_prefix): + ruby.install_environment(fake_gem_prefix, C.DEFAULT, ()) # Should have created rbenv directory - assert os.path.exists(prefix.path('rbenv-default')) + assert os.path.exists(fake_gem_prefix.path('rbenv-default')) # Should be able to activate using our script and access rbenv - with ruby.in_env(prefix, 'default'): + with ruby.in_env(fake_gem_prefix, 'default'): cmd_output('rbenv', '--help') @xfailif_windows # pragma: win32 no cover -def test_install_rbenv_with_version(tempdir_factory): - prefix = Prefix(tempdir_factory.get()) - ruby._install_rbenv(prefix, version='1.9.3p547') +def test_install_ruby_with_version(fake_gem_prefix): + ruby.install_environment(fake_gem_prefix, '2.7.2', ()) # Should be able to activate and use rbenv install - with ruby.in_env(prefix, '1.9.3p547'): + with ruby.in_env(fake_gem_prefix, '2.7.2'): cmd_output('rbenv', 'install', '--help') diff --git a/tests/main_test.py b/tests/main_test.py index f7abeeb..6738df6 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -6,7 +6,7 @@ import pytest import pre_commit.constants as C from pre_commit import main -from pre_commit.error_handler import FatalError +from pre_commit.errors import FatalError from testing.auto_namedtuple import auto_namedtuple diff --git a/tests/repository_test.py b/tests/repository_test.py index 84e4da9..3d5093d 100644 --- a/tests/repository_test.py +++ b/tests/repository_test.py @@ -1,5 +1,4 @@ import os.path -import re import shutil import sys from typing import Any @@ -8,6 +7,7 @@ from unittest import mock import cfgv import pytest +import re_assert import pre_commit.constants as C from pre_commit.clientlib import CONFIG_SCHEMA @@ -31,6 +31,7 @@ from testing.fixtures import make_repo from testing.fixtures import modify_manifest from testing.util import cwd from testing.util import get_resource_path +from testing.util import skipif_cant_run_coursier from testing.util import skipif_cant_run_docker from testing.util import skipif_cant_run_swift from testing.util import xfailif_windows @@ -94,8 +95,8 @@ def test_conda_with_additional_dependencies_hook(tempdir_factory, store): config_kwargs={ 'hooks': [{ 'id': 'additional-deps', - 'args': ['-c', 'import mccabe; print("OK")'], - 'additional_dependencies': ['mccabe'], + 'args': ['-c', 'import tzdata; print("OK")'], + 'additional_dependencies': ['python-tzdata'], }], }, ) @@ -109,8 +110,8 @@ def test_local_conda_additional_dependencies(store): 'name': 'local-conda', 'entry': 'python', 'language': 'conda', - 'args': ['-c', 'import mccabe; print("OK")'], - 'additional_dependencies': ['mccabe'], + 'args': ['-c', 'import tzdata; print("OK")'], + 'additional_dependencies': ['python-tzdata'], }], } hook = _get_hook(config, store, 'local-conda') @@ -195,6 +196,15 @@ def test_versioned_python_hook(tempdir_factory, store): ) +@skipif_cant_run_coursier # pragma: win32 no cover +def test_run_a_coursier_hook(tempdir_factory, store): + _test_hook_repo( + tempdir_factory, store, 'coursier_hooks_repo', + 'echo-java', + ['Hello World from coursier'], b'Hello World from coursier\n', + ) + + @skipif_cant_run_docker # pragma: win32 no cover def test_run_a_docker_hook(tempdir_factory, store): _test_hook_repo( @@ -843,12 +853,12 @@ def test_too_new_version(tempdir_factory, store, fake_log_handler): with pytest.raises(SystemExit): _get_hook(config, store, 'bash_hook') msg = fake_log_handler.handle.call_args[0][0].msg - assert re.match( + pattern = re_assert.Matches( r'^The hook `bash_hook` requires pre-commit version 999\.0\.0 but ' r'version \d+\.\d+\.\d+ is installed. ' r'Perhaps run `pip install --upgrade pre-commit`\.$', - msg, ) + pattern.assert_matches(msg) @pytest.mark.parametrize('version', ('0.1.0', C.VERSION)) @@ -917,3 +927,17 @@ def test_local_perl_additional_dependencies(store): ret, out = _hook_run(hook, (), color=False) assert ret == 0 assert _norm_out(out).startswith(b'This is perltidy, v20200110') + + +@pytest.mark.parametrize( + 'repo', + ( + 'dotnet_hooks_csproj_repo', + 'dotnet_hooks_sln_repo', + ), +) +def test_dotnet_hook(tempdir_factory, store, repo): + _test_hook_repo( + tempdir_factory, store, repo, + 'dotnet example hook', [], b'Hello from dotnet!\n', + ) diff --git a/tests/xargs_test.py b/tests/xargs_test.py index 1fc9207..4f6136e 100644 --- a/tests/xargs_test.py +++ b/tests/xargs_test.py @@ -160,7 +160,7 @@ def test_xargs_concurrency(): assert ret == 0 pids = stdout.splitlines() assert len(pids) == 5 - # It would take 0.5*5=2.5 seconds ot run all of these in serial, so if it + # It would take 0.5*5=2.5 seconds to run all of these in serial, so if it # takes less, they must have run concurrently. assert elapsed < 2.5 @@ -3,7 +3,7 @@ envlist = py36,py37,py38,pypy3,pre-commit [testenv] deps = -rrequirements-dev.txt -passenv = HOME LOCALAPPDATA RUSTUP_HOME +passenv = APPDATA HOME LOCALAPPDATA PROGRAMFILES RUSTUP_HOME commands = coverage erase coverage run -m pytest {posargs:tests} |