diff options
| -rw-r--r-- | .github/workflows/languages.yaml | 2 | ||||
| -rw-r--r-- | .pre-commit-config.yaml | 15 | ||||
| -rw-r--r-- | CHANGELOG.md | 15 | ||||
| -rw-r--r-- | CONTRIBUTING.md | 8 | ||||
| -rw-r--r-- | pre_commit/all_languages.py | 2 | ||||
| -rw-r--r-- | pre_commit/commands/install_uninstall.py | 3 | ||||
| -rw-r--r-- | pre_commit/languages/haskell.py | 56 | ||||
| -rw-r--r-- | pre_commit/xargs.py | 11 | ||||
| -rw-r--r-- | setup.cfg | 2 | ||||
| -rw-r--r-- | tests/lang_base_test.py | 27 | ||||
| -rw-r--r-- | tests/languages/golang_test.py | 2 | ||||
| -rw-r--r-- | tests/languages/haskell_test.py | 50 | ||||
| -rw-r--r-- | tests/xargs_test.py | 9 |
13 files changed, 183 insertions, 19 deletions
diff --git a/.github/workflows/languages.yaml b/.github/workflows/languages.yaml index 7e97158..5a6ae9c 100644 --- a/.github/workflows/languages.yaml +++ b/.github/workflows/languages.yaml @@ -63,6 +63,8 @@ jobs: echo 'C:\Strawberry\c\bin' >> "$GITHUB_PATH" shell: bash if: matrix.os == 'windows-latest' && matrix.language == 'perl' + - uses: haskell/actions/setup@v2 + if: matrix.language == 'haskell' - name: install deps run: python -mpip install -e . -r requirements-dev.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7810d22..5c6f62b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,35 +10,34 @@ repos: - id: name-tests-test - id: requirements-txt-fixer - repo: https://github.com/asottile/setup-cfg-fmt - rev: v2.3.0 + rev: v2.4.0 hooks: - id: setup-cfg-fmt - repo: https://github.com/asottile/reorder-python-imports - rev: v3.9.0 + rev: v3.10.0 hooks: - id: reorder-python-imports exclude: ^(pre_commit/resources/|testing/resources/python3_hooks_repo/) args: [--py38-plus, --add-import, 'from __future__ import annotations'] - repo: https://github.com/asottile/add-trailing-comma - rev: v2.5.1 + rev: v3.0.1 hooks: - id: add-trailing-comma - args: [--py36-plus] - repo: https://github.com/asottile/pyupgrade - rev: v3.6.0 + rev: v3.10.1 hooks: - id: pyupgrade args: [--py38-plus] - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v2.0.2 + rev: v2.0.4 hooks: - id: autopep8 - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.3.0 + rev: v1.5.1 hooks: - id: mypy additional_dependencies: [types-all] diff --git a/CHANGELOG.md b/CHANGELOG.md index 722e8ff..9e2ef0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +3.4.0 - 2023-09-02 +================== + +### Features +- Add `language: haskell`. + - #2932 by @alunduil. +- Improve cpu count detection when run under cgroups. + - #2979 PR by @jdb8. + - #2978 issue by @jdb8. + +### Fixes +- Handle negative exit codes from hooks receiving posix signals. + - #2971 PR by @chriskuehl. + - #2970 issue by @chriskuehl. + 3.3.3 - 2023-06-13 ================== diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ab3a929..da7f943 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,7 +92,7 @@ language, for example: here are the apis that should be implemented for a language -Note that these are also documented in [`pre_commit/languages/all.py`](https://github.com/pre-commit/pre-commit/blob/main/pre_commit/languages/all.py) +Note that these are also documented in [`pre_commit/lang_base.py`](https://github.com/pre-commit/pre-commit/blob/main/pre_commit/lang_base.py) #### `ENVIRONMENT_DIR` @@ -111,7 +111,7 @@ one cannot be determined, return `'default'`. You generally don't need to implement this on a first pass and can just use: ```python -get_default_version = helpers.basic_default_version +get_default_version = lang_base.basic_default_version ``` `python` is currently the only language which implements this api @@ -125,7 +125,7 @@ healthy. You generally don't need to implement this on a first pass and can just use: ```python -health_check = helpers.basic_healthy_check +health_check = lang_base.basic_health_check ``` `python` is currently the only language which implements this api, for python @@ -137,7 +137,7 @@ this is the trickiest one to implement and where all the smart parts happen. this api should do the following things -- (0th / 3rd class): `install_environment = helpers.no_install` +- (0th / 3rd class): `install_environment = lang_base.no_install` - (1st class): install a language runtime into the hook's directory - (2nd class): install the package at `.` into the `ENVIRONMENT_DIR` - (2nd class, optional): install packages listed in `additional_dependencies` diff --git a/pre_commit/all_languages.py b/pre_commit/all_languages.py index 2bed706..476bad9 100644 --- a/pre_commit/all_languages.py +++ b/pre_commit/all_languages.py @@ -9,6 +9,7 @@ from pre_commit.languages import docker_image from pre_commit.languages import dotnet from pre_commit.languages import fail from pre_commit.languages import golang +from pre_commit.languages import haskell from pre_commit.languages import lua from pre_commit.languages import node from pre_commit.languages import perl @@ -31,6 +32,7 @@ languages: dict[str, Language] = { 'dotnet': dotnet, 'fail': fail, 'golang': golang, + 'haskell': haskell, 'lua': lua, 'node': node, 'perl': perl, diff --git a/pre_commit/commands/install_uninstall.py b/pre_commit/commands/install_uninstall.py index 5ff6cba..d19e0d4 100644 --- a/pre_commit/commands/install_uninstall.py +++ b/pre_commit/commands/install_uninstall.py @@ -103,8 +103,7 @@ def _install_hook_script( hook_file.write(before + TEMPLATE_START) hook_file.write(f'INSTALL_PYTHON={shlex.quote(sys.executable)}\n') - # TODO: python3.8+: shlex.join - args_s = ' '.join(shlex.quote(part) for part in args) + args_s = shlex.join(args) hook_file.write(f'ARGS=({args_s})\n') hook_file.write(TEMPLATE_END + after) make_executable(hook_path) diff --git a/pre_commit/languages/haskell.py b/pre_commit/languages/haskell.py new file mode 100644 index 0000000..76442eb --- /dev/null +++ b/pre_commit/languages/haskell.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import contextlib +import os.path +from typing import Generator +from typing import Sequence + +from pre_commit import lang_base +from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT +from pre_commit.envcontext import Var +from pre_commit.errors import FatalError +from pre_commit.prefix import Prefix + +ENVIRONMENT_DIR = 'hs_env' +get_default_version = lang_base.basic_get_default_version +health_check = lang_base.basic_health_check +run_hook = lang_base.basic_run_hook + + +def get_env_patch(target_dir: str) -> PatchesT: + bin_path = os.path.join(target_dir, 'bin') + return (('PATH', (bin_path, os.pathsep, Var('PATH'))),) + + +@contextlib.contextmanager +def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: + envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) + with envcontext(get_env_patch(envdir)): + yield + + +def install_environment( + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], +) -> None: + lang_base.assert_version_default('haskell', version) + envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) + + pkgs = [*prefix.star('.cabal'), *additional_dependencies] + if not pkgs: + raise FatalError('Expected .cabal files or additional_dependencies') + + bindir = os.path.join(envdir, 'bin') + os.makedirs(bindir, exist_ok=True) + lang_base.setup_cmd(prefix, ('cabal', 'update')) + lang_base.setup_cmd( + prefix, + ( + 'cabal', 'install', + '--install-method', 'copy', + '--installdir', bindir, + *pkgs, + ), + ) diff --git a/pre_commit/xargs.py b/pre_commit/xargs.py index 31be6f3..a7493c0 100644 --- a/pre_commit/xargs.py +++ b/pre_commit/xargs.py @@ -25,6 +25,14 @@ TRet = TypeVar('TRet') def cpu_count() -> int: try: + # On systems that support it, this will return a more accurate count of + # usable CPUs for the current process, which will take into account + # cgroup limits + return len(os.sched_getaffinity(0)) + except AttributeError: + pass + + try: return multiprocessing.cpu_count() except NotImplementedError: return 1 @@ -170,7 +178,8 @@ def xargs( results = thread_map(run_cmd_partition, partitions) for proc_retcode, proc_out, _ in results: - retcode = max(retcode, proc_retcode) + if abs(proc_retcode) > abs(retcode): + retcode = proc_retcode stdout += proc_out return retcode, stdout @@ -1,6 +1,6 @@ [metadata] name = pre_commit -version = 3.3.3 +version = 3.4.0 description = A framework for managing and maintaining multi-language pre-commit hooks. long_description = file: README.md long_description_content_type = text/markdown diff --git a/tests/lang_base_test.py b/tests/lang_base_test.py index a532b6a..1cffa0e 100644 --- a/tests/lang_base_test.py +++ b/tests/lang_base_test.py @@ -30,6 +30,19 @@ def homedir_mck(): yield +@pytest.fixture +def no_sched_getaffinity(): + # Simulates an OS without os.sched_getaffinity available (mac/windows) + # https://docs.python.org/3/library/os.html#interface-to-the-scheduler + with mock.patch.object( + os, + 'sched_getaffinity', + create=True, + side_effect=AttributeError, + ): + yield + + def test_exe_exists_does_not_exist(find_exe_mck, homedir_mck): find_exe_mck.return_value = None assert lang_base.exe_exists('ruby') is False @@ -116,7 +129,17 @@ def test_no_env_noop(tmp_path): assert before == inside == after -def test_target_concurrency_normal(): +def test_target_concurrency_sched_getaffinity(no_sched_getaffinity): + with mock.patch.object( + os, + 'sched_getaffinity', + return_value=set(range(345)), + ): + with mock.patch.dict(os.environ, clear=True): + assert lang_base.target_concurrency() == 345 + + +def test_target_concurrency_without_sched_getaffinity(no_sched_getaffinity): with mock.patch.object(multiprocessing, 'cpu_count', return_value=123): with mock.patch.dict(os.environ, {}, clear=True): assert lang_base.target_concurrency() == 123 @@ -134,7 +157,7 @@ def test_target_concurrency_on_travis(): assert lang_base.target_concurrency() == 2 -def test_target_concurrency_cpu_count_not_implemented(): +def test_target_concurrency_cpu_count_not_implemented(no_sched_getaffinity): with mock.patch.object( multiprocessing, 'cpu_count', side_effect=NotImplementedError, ): diff --git a/tests/languages/golang_test.py b/tests/languages/golang_test.py index ec5a878..6406267 100644 --- a/tests/languages/golang_test.py +++ b/tests/languages/golang_test.py @@ -128,7 +128,7 @@ def test_local_golang_additional_deps(tmp_path): deps=('golang.org/x/example/hello@latest',), ) - assert ret == (0, b'Hello, Go examples!\n') + assert ret == (0, b'Hello, world!\n') def test_golang_hook_still_works_when_gobin_is_set(tmp_path): diff --git a/tests/languages/haskell_test.py b/tests/languages/haskell_test.py new file mode 100644 index 0000000..f888109 --- /dev/null +++ b/tests/languages/haskell_test.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import pytest + +from pre_commit.errors import FatalError +from pre_commit.languages import haskell +from pre_commit.util import win_exe +from testing.language_helpers import run_language + + +def test_run_example_executable(tmp_path): + example_cabal = '''\ +cabal-version: 2.4 +name: example +version: 0.1.0.0 + +executable example + main-is: Main.hs + + build-depends: base >=4 + default-language: Haskell2010 +''' + main_hs = '''\ +module Main where + +main :: IO () +main = putStrLn "Hello, Haskell!" +''' + tmp_path.joinpath('example.cabal').write_text(example_cabal) + tmp_path.joinpath('Main.hs').write_text(main_hs) + + result = run_language(tmp_path, haskell, 'example') + assert result == (0, b'Hello, Haskell!\n') + + # should not symlink things into environments + exe = tmp_path.joinpath(win_exe('hs_env-default/bin/example')) + assert exe.is_file() + assert not exe.is_symlink() + + +def test_run_dep(tmp_path): + result = run_language(tmp_path, haskell, 'hello', deps=['hello']) + assert result == (0, b'Hello, World!\n') + + +def test_run_empty(tmp_path): + with pytest.raises(FatalError) as excinfo: + run_language(tmp_path, haskell, 'example') + msg, = excinfo.value.args + assert msg == 'Expected .cabal files or additional_dependencies' diff --git a/tests/xargs_test.py b/tests/xargs_test.py index 7c41f98..b0a8e0d 100644 --- a/tests/xargs_test.py +++ b/tests/xargs_test.py @@ -147,6 +147,15 @@ def test_xargs_retcode_normal(): assert ret == 5 +@pytest.mark.xfail(sys.platform == 'win32', reason='posix only') +def test_xargs_retcode_killed_by_signal(): + ret, _ = xargs.xargs( + parse_shebang.normalize_cmd(('bash', '-c', 'kill -9 $$', '--')), + ('foo', 'bar'), + ) + assert ret == -9 + + def test_xargs_concurrency(): bash_cmd = parse_shebang.normalize_cmd(('bash', '-c')) print_pid = ('sleep 0.5 && echo $$',) |