summaryrefslogtreecommitdiffstats
path: root/pre_commit
diff options
context:
space:
mode:
Diffstat (limited to 'pre_commit')
-rw-r--r--pre_commit/__init__.py0
-rw-r--r--pre_commit/__main__.py5
-rw-r--r--pre_commit/clientlib.py317
-rw-r--r--pre_commit/color.py97
-rw-r--r--pre_commit/commands/__init__.py0
-rw-r--r--pre_commit/commands/autoupdate.py182
-rw-r--r--pre_commit/commands/clean.py14
-rw-r--r--pre_commit/commands/gc.py90
-rw-r--r--pre_commit/commands/hook_impl.py187
-rw-r--r--pre_commit/commands/init_templatedir.py33
-rw-r--r--pre_commit/commands/install_uninstall.py175
-rw-r--r--pre_commit/commands/migrate_config.py59
-rw-r--r--pre_commit/commands/run.py360
-rw-r--r--pre_commit/commands/sample_config.py21
-rw-r--r--pre_commit/commands/try_repo.py77
-rw-r--r--pre_commit/constants.py24
-rw-r--r--pre_commit/envcontext.py67
-rw-r--r--pre_commit/error_handler.py64
-rw-r--r--pre_commit/file_lock.py76
-rw-r--r--pre_commit/git.py196
-rw-r--r--pre_commit/hook.py63
-rw-r--r--pre_commit/languages/__init__.py0
-rw-r--r--pre_commit/languages/all.py60
-rw-r--r--pre_commit/languages/conda.py84
-rw-r--r--pre_commit/languages/docker.py114
-rw-r--r--pre_commit/languages/docker_image.py22
-rw-r--r--pre_commit/languages/fail.py20
-rw-r--r--pre_commit/languages/golang.py97
-rw-r--r--pre_commit/languages/helpers.py109
-rw-r--r--pre_commit/languages/node.py93
-rw-r--r--pre_commit/languages/perl.py67
-rw-r--r--pre_commit/languages/pygrep.py87
-rw-r--r--pre_commit/languages/python.py210
-rw-r--r--pre_commit/languages/python_venv.py46
-rw-r--r--pre_commit/languages/ruby.py126
-rw-r--r--pre_commit/languages/rust.py106
-rw-r--r--pre_commit/languages/script.py19
-rw-r--r--pre_commit/languages/swift.py64
-rw-r--r--pre_commit/languages/system.py19
-rw-r--r--pre_commit/logging_handler.py40
-rw-r--r--pre_commit/main.py410
-rw-r--r--pre_commit/make_archives.py65
-rw-r--r--pre_commit/meta_hooks/__init__.py0
-rw-r--r--pre_commit/meta_hooks/check_hooks_apply.py39
-rw-r--r--pre_commit/meta_hooks/check_useless_excludes.py72
-rw-r--r--pre_commit/meta_hooks/identity.py16
-rw-r--r--pre_commit/output.py32
-rw-r--r--pre_commit/parse_shebang.py84
-rw-r--r--pre_commit/prefix.py17
-rw-r--r--pre_commit/repository.py208
-rw-r--r--pre_commit/resources/__init__.py0
-rw-r--r--pre_commit/resources/empty_template_.npmignore1
-rw-r--r--pre_commit/resources/empty_template_Cargo.toml7
-rw-r--r--pre_commit/resources/empty_template_Makefile.PL6
-rw-r--r--pre_commit/resources/empty_template_environment.yml9
-rw-r--r--pre_commit/resources/empty_template_main.go3
-rw-r--r--pre_commit/resources/empty_template_main.rs1
-rw-r--r--pre_commit/resources/empty_template_package.json4
-rw-r--r--pre_commit/resources/empty_template_pre_commit_dummy_package.gemspec6
-rw-r--r--pre_commit/resources/empty_template_setup.py4
-rwxr-xr-xpre_commit/resources/hook-tmpl44
-rw-r--r--pre_commit/resources/rbenv.tar.gzbin0 -> 31781 bytes
-rw-r--r--pre_commit/resources/ruby-build.tar.gzbin0 -> 62567 bytes
-rw-r--r--pre_commit/resources/ruby-download.tar.gzbin0 -> 5343 bytes
-rw-r--r--pre_commit/staged_files_only.py90
-rw-r--r--pre_commit/store.py250
-rw-r--r--pre_commit/util.py272
-rw-r--r--pre_commit/xargs.py157
68 files changed, 5287 insertions, 0 deletions
diff --git a/pre_commit/__init__.py b/pre_commit/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/pre_commit/__init__.py
diff --git a/pre_commit/__main__.py b/pre_commit/__main__.py
new file mode 100644
index 0000000..5414068
--- /dev/null
+++ b/pre_commit/__main__.py
@@ -0,0 +1,5 @@
+from pre_commit.main import main
+
+
+if __name__ == '__main__':
+ exit(main())
diff --git a/pre_commit/clientlib.py b/pre_commit/clientlib.py
new file mode 100644
index 0000000..56ec0dd
--- /dev/null
+++ b/pre_commit/clientlib.py
@@ -0,0 +1,317 @@
+import argparse
+import functools
+import logging
+import shlex
+import sys
+from typing import Any
+from typing import Dict
+from typing import Optional
+from typing import Sequence
+
+import cfgv
+from identify.identify import ALL_TAGS
+
+import pre_commit.constants as C
+from pre_commit.error_handler import FatalError
+from pre_commit.languages.all import all_languages
+from pre_commit.util import parse_version
+from pre_commit.util import yaml_load
+
+logger = logging.getLogger('pre_commit')
+
+check_string_regex = cfgv.check_and(cfgv.check_string, cfgv.check_regex)
+
+
+def check_type_tag(tag: str) -> None:
+ if tag not in ALL_TAGS:
+ raise cfgv.ValidationError(
+ f'Type tag {tag!r} is not recognized. '
+ f'Try upgrading identify and pre-commit?',
+ )
+
+
+def check_min_version(version: str) -> None:
+ if parse_version(version) > parse_version(C.VERSION):
+ raise cfgv.ValidationError(
+ f'pre-commit version {version} is required but version '
+ f'{C.VERSION} is installed. '
+ f'Perhaps run `pip install --upgrade pre-commit`.',
+ )
+
+
+def _make_argparser(filenames_help: str) -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser()
+ parser.add_argument('filenames', nargs='*', help=filenames_help)
+ parser.add_argument('-V', '--version', action='version', version=C.VERSION)
+ return parser
+
+
+MANIFEST_HOOK_DICT = cfgv.Map(
+ 'Hook', 'id',
+
+ cfgv.Required('id', cfgv.check_string),
+ cfgv.Required('name', cfgv.check_string),
+ cfgv.Required('entry', cfgv.check_string),
+ cfgv.Required('language', cfgv.check_one_of(all_languages)),
+ cfgv.Optional('alias', cfgv.check_string, ''),
+
+ cfgv.Optional('files', check_string_regex, ''),
+ cfgv.Optional('exclude', check_string_regex, '^$'),
+ cfgv.Optional('types', cfgv.check_array(check_type_tag), ['file']),
+ cfgv.Optional('exclude_types', cfgv.check_array(check_type_tag), []),
+
+ cfgv.Optional(
+ 'additional_dependencies', cfgv.check_array(cfgv.check_string), [],
+ ),
+ cfgv.Optional('args', cfgv.check_array(cfgv.check_string), []),
+ cfgv.Optional('always_run', cfgv.check_bool, False),
+ cfgv.Optional('pass_filenames', cfgv.check_bool, True),
+ cfgv.Optional('description', cfgv.check_string, ''),
+ cfgv.Optional('language_version', cfgv.check_string, C.DEFAULT),
+ cfgv.Optional('log_file', cfgv.check_string, ''),
+ cfgv.Optional('minimum_pre_commit_version', cfgv.check_string, '0'),
+ cfgv.Optional('require_serial', cfgv.check_bool, False),
+ cfgv.Optional('stages', cfgv.check_array(cfgv.check_one_of(C.STAGES)), []),
+ cfgv.Optional('verbose', cfgv.check_bool, False),
+)
+MANIFEST_SCHEMA = cfgv.Array(MANIFEST_HOOK_DICT)
+
+
+class InvalidManifestError(FatalError):
+ pass
+
+
+load_manifest = functools.partial(
+ cfgv.load_from_filename,
+ schema=MANIFEST_SCHEMA,
+ load_strategy=yaml_load,
+ exc_tp=InvalidManifestError,
+)
+
+
+def validate_manifest_main(argv: Optional[Sequence[str]] = None) -> int:
+ parser = _make_argparser('Manifest filenames.')
+ args = parser.parse_args(argv)
+ ret = 0
+ for filename in args.filenames:
+ try:
+ load_manifest(filename)
+ except InvalidManifestError as e:
+ print(e)
+ ret = 1
+ return ret
+
+
+LOCAL = 'local'
+META = 'meta'
+
+
+class MigrateShaToRev:
+ key = 'rev'
+
+ @staticmethod
+ def _cond(key: str) -> cfgv.Conditional:
+ return cfgv.Conditional(
+ key, cfgv.check_string,
+ condition_key='repo',
+ condition_value=cfgv.NotIn(LOCAL, META),
+ ensure_absent=True,
+ )
+
+ def check(self, dct: Dict[str, Any]) -> None:
+ if dct.get('repo') in {LOCAL, META}:
+ self._cond('rev').check(dct)
+ self._cond('sha').check(dct)
+ elif 'sha' in dct and 'rev' in dct:
+ raise cfgv.ValidationError('Cannot specify both sha and rev')
+ elif 'sha' in dct:
+ self._cond('sha').check(dct)
+ else:
+ self._cond('rev').check(dct)
+
+ def apply_default(self, dct: Dict[str, Any]) -> None:
+ if 'sha' in dct:
+ dct['rev'] = dct.pop('sha')
+
+ remove_default = cfgv.Required.remove_default
+
+
+def _entry(modname: str) -> str:
+ """the hook `entry` is passed through `shlex.split()` by the command
+ runner, so to prevent issues with spaces and backslashes (on Windows)
+ it must be quoted here.
+ """
+ return f'{shlex.quote(sys.executable)} -m pre_commit.meta_hooks.{modname}'
+
+
+def warn_unknown_keys_root(
+ extra: Sequence[str],
+ orig_keys: Sequence[str],
+ dct: Dict[str, str],
+) -> None:
+ logger.warning(f'Unexpected key(s) present at root: {", ".join(extra)}')
+
+
+def warn_unknown_keys_repo(
+ extra: Sequence[str],
+ orig_keys: Sequence[str],
+ dct: Dict[str, str],
+) -> None:
+ logger.warning(
+ f'Unexpected key(s) present on {dct["repo"]}: {", ".join(extra)}',
+ )
+
+
+_meta = (
+ (
+ 'check-hooks-apply', (
+ ('name', 'Check hooks apply to the repository'),
+ ('files', C.CONFIG_FILE),
+ ('entry', _entry('check_hooks_apply')),
+ ),
+ ),
+ (
+ 'check-useless-excludes', (
+ ('name', 'Check for useless excludes'),
+ ('files', C.CONFIG_FILE),
+ ('entry', _entry('check_useless_excludes')),
+ ),
+ ),
+ (
+ 'identity', (
+ ('name', 'identity'),
+ ('verbose', True),
+ ('entry', _entry('identity')),
+ ),
+ ),
+)
+
+META_HOOK_DICT = cfgv.Map(
+ 'Hook', 'id',
+ cfgv.Required('id', cfgv.check_string),
+ cfgv.Required('id', cfgv.check_one_of(tuple(k for k, _ in _meta))),
+ # language must be system
+ cfgv.Optional('language', cfgv.check_one_of({'system'}), 'system'),
+ *(
+ # default to the hook definition for the meta hooks
+ cfgv.ConditionalOptional(key, cfgv.check_any, value, 'id', hook_id)
+ for hook_id, values in _meta
+ for key, value in values
+ ),
+ *(
+ # default to the "manifest" parsing
+ cfgv.OptionalNoDefault(item.key, item.check_fn)
+ # these will always be defaulted above
+ if item.key in {'name', 'language', 'entry'} else
+ item
+ for item in MANIFEST_HOOK_DICT.items
+ ),
+)
+CONFIG_HOOK_DICT = cfgv.Map(
+ 'Hook', 'id',
+
+ cfgv.Required('id', cfgv.check_string),
+
+ # All keys in manifest hook dict are valid in a config hook dict, but
+ # are optional.
+ # No defaults are provided here as the config is merged on top of the
+ # manifest.
+ *(
+ cfgv.OptionalNoDefault(item.key, item.check_fn)
+ for item in MANIFEST_HOOK_DICT.items
+ if item.key != 'id'
+ ),
+)
+CONFIG_REPO_DICT = cfgv.Map(
+ 'Repository', 'repo',
+
+ cfgv.Required('repo', cfgv.check_string),
+
+ cfgv.ConditionalRecurse(
+ 'hooks', cfgv.Array(CONFIG_HOOK_DICT),
+ 'repo', cfgv.NotIn(LOCAL, META),
+ ),
+ cfgv.ConditionalRecurse(
+ 'hooks', cfgv.Array(MANIFEST_HOOK_DICT),
+ 'repo', LOCAL,
+ ),
+ cfgv.ConditionalRecurse(
+ 'hooks', cfgv.Array(META_HOOK_DICT),
+ 'repo', META,
+ ),
+
+ MigrateShaToRev(),
+ cfgv.WarnAdditionalKeys(('repo', 'rev', 'hooks'), warn_unknown_keys_repo),
+)
+DEFAULT_LANGUAGE_VERSION = cfgv.Map(
+ 'DefaultLanguageVersion', None,
+ cfgv.NoAdditionalKeys(all_languages),
+ *(cfgv.Optional(x, cfgv.check_string, C.DEFAULT) for x in all_languages),
+)
+CONFIG_SCHEMA = cfgv.Map(
+ 'Config', None,
+
+ cfgv.RequiredRecurse('repos', cfgv.Array(CONFIG_REPO_DICT)),
+ cfgv.OptionalRecurse(
+ 'default_language_version', DEFAULT_LANGUAGE_VERSION, {},
+ ),
+ cfgv.Optional(
+ 'default_stages',
+ cfgv.check_array(cfgv.check_one_of(C.STAGES)),
+ C.STAGES,
+ ),
+ cfgv.Optional('files', check_string_regex, ''),
+ cfgv.Optional('exclude', check_string_regex, '^$'),
+ cfgv.Optional('fail_fast', cfgv.check_bool, False),
+ cfgv.Optional(
+ 'minimum_pre_commit_version',
+ cfgv.check_and(cfgv.check_string, check_min_version),
+ '0',
+ ),
+ cfgv.WarnAdditionalKeys(
+ (
+ 'repos',
+ 'default_language_version',
+ 'default_stages',
+ 'files',
+ 'exclude',
+ 'fail_fast',
+ 'minimum_pre_commit_version',
+ ),
+ warn_unknown_keys_root,
+ ),
+)
+
+
+class InvalidConfigError(FatalError):
+ pass
+
+
+def ordered_load_normalize_legacy_config(contents: str) -> Dict[str, Any]:
+ data = yaml_load(contents)
+ if isinstance(data, list):
+ # TODO: Once happy, issue a deprecation warning and instructions
+ return {'repos': data}
+ else:
+ return data
+
+
+load_config = functools.partial(
+ cfgv.load_from_filename,
+ schema=CONFIG_SCHEMA,
+ load_strategy=ordered_load_normalize_legacy_config,
+ exc_tp=InvalidConfigError,
+)
+
+
+def validate_config_main(argv: Optional[Sequence[str]] = None) -> int:
+ parser = _make_argparser('Config filenames.')
+ args = parser.parse_args(argv)
+ ret = 0
+ for filename in args.filenames:
+ try:
+ load_config(filename)
+ except InvalidConfigError as e:
+ print(e)
+ ret = 1
+ return ret
diff --git a/pre_commit/color.py b/pre_commit/color.py
new file mode 100644
index 0000000..5fa7042
--- /dev/null
+++ b/pre_commit/color.py
@@ -0,0 +1,97 @@
+import os
+import sys
+
+if sys.platform == 'win32': # pragma: no cover (windows)
+ def _enable() -> None:
+ from ctypes import POINTER
+ from ctypes import windll
+ from ctypes import WinError
+ from ctypes import WINFUNCTYPE
+ from ctypes.wintypes import BOOL
+ from ctypes.wintypes import DWORD
+ from ctypes.wintypes import HANDLE
+
+ STD_OUTPUT_HANDLE = -11
+ ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4
+
+ def bool_errcheck(result, func, args):
+ if not result:
+ raise WinError()
+ return args
+
+ GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)(
+ ('GetStdHandle', windll.kernel32), ((1, 'nStdHandle'),),
+ )
+
+ GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))(
+ ('GetConsoleMode', windll.kernel32),
+ ((1, 'hConsoleHandle'), (2, 'lpMode')),
+ )
+ GetConsoleMode.errcheck = bool_errcheck
+
+ SetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, DWORD)(
+ ('SetConsoleMode', windll.kernel32),
+ ((1, 'hConsoleHandle'), (1, 'dwMode')),
+ )
+ SetConsoleMode.errcheck = bool_errcheck
+
+ # As of Windows 10, the Windows console supports (some) ANSI escape
+ # sequences, but it needs to be enabled using `SetConsoleMode` first.
+ #
+ # More info on the escape sequences supported:
+ # https://msdn.microsoft.com/en-us/library/windows/desktop/mt638032(v=vs.85).aspx
+ stdout = GetStdHandle(STD_OUTPUT_HANDLE)
+ flags = GetConsoleMode(stdout)
+ SetConsoleMode(stdout, flags | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
+
+ try:
+ _enable()
+ except OSError:
+ terminal_supports_color = False
+ else:
+ terminal_supports_color = True
+else: # pragma: win32 no cover
+ terminal_supports_color = True
+
+RED = '\033[41m'
+GREEN = '\033[42m'
+YELLOW = '\033[43;30m'
+TURQUOISE = '\033[46;30m'
+SUBTLE = '\033[2m'
+NORMAL = '\033[m'
+
+
+def format_color(text: str, color: str, use_color_setting: bool) -> str:
+ """Format text with color.
+
+ Args:
+ text - Text to be formatted with color if `use_color`
+ color - The color start string
+ use_color_setting - Whether or not to color
+ """
+ if use_color_setting:
+ return f'{color}{text}{NORMAL}'
+ else:
+ return text
+
+
+COLOR_CHOICES = ('auto', 'always', 'never')
+
+
+def use_color(setting: str) -> bool:
+ """Choose whether to use color based on the command argument.
+
+ Args:
+ setting - Either `auto`, `always`, or `never`
+ """
+ if setting not in COLOR_CHOICES:
+ raise ValueError(setting)
+
+ return (
+ setting == 'always' or (
+ setting == 'auto' and
+ sys.stdout.isatty() and
+ terminal_supports_color and
+ os.getenv('TERM') != 'dumb'
+ )
+ )
diff --git a/pre_commit/commands/__init__.py b/pre_commit/commands/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/pre_commit/commands/__init__.py
diff --git a/pre_commit/commands/autoupdate.py b/pre_commit/commands/autoupdate.py
new file mode 100644
index 0000000..5a9a988
--- /dev/null
+++ b/pre_commit/commands/autoupdate.py
@@ -0,0 +1,182 @@
+import os.path
+import re
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import NamedTuple
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+
+import pre_commit.constants as C
+from pre_commit import git
+from pre_commit import output
+from pre_commit.clientlib import InvalidManifestError
+from pre_commit.clientlib import load_config
+from pre_commit.clientlib import load_manifest
+from pre_commit.clientlib import LOCAL
+from pre_commit.clientlib import META
+from pre_commit.commands.migrate_config import migrate_config
+from pre_commit.store import Store
+from pre_commit.util import CalledProcessError
+from pre_commit.util import cmd_output
+from pre_commit.util import cmd_output_b
+from pre_commit.util import tmpdir
+from pre_commit.util import yaml_dump
+from pre_commit.util import yaml_load
+
+
+class RevInfo(NamedTuple):
+ repo: str
+ rev: str
+ frozen: Optional[str]
+
+ @classmethod
+ def from_config(cls, config: Dict[str, Any]) -> 'RevInfo':
+ return cls(config['repo'], config['rev'], None)
+
+ def update(self, tags_only: bool, freeze: bool) -> 'RevInfo':
+ if tags_only:
+ tag_cmd = ('git', 'describe', 'FETCH_HEAD', '--tags', '--abbrev=0')
+ else:
+ tag_cmd = ('git', 'describe', 'FETCH_HEAD', '--tags', '--exact')
+
+ with tmpdir() as tmp:
+ git.init_repo(tmp, self.repo)
+ cmd_output_b('git', 'fetch', 'origin', 'HEAD', '--tags', cwd=tmp)
+
+ try:
+ rev = cmd_output(*tag_cmd, cwd=tmp)[1].strip()
+ except CalledProcessError:
+ cmd = ('git', 'rev-parse', 'FETCH_HEAD')
+ rev = cmd_output(*cmd, cwd=tmp)[1].strip()
+
+ frozen = None
+ if freeze:
+ exact = cmd_output('git', 'rev-parse', rev, cwd=tmp)[1].strip()
+ if exact != rev:
+ rev, frozen = exact, rev
+ return self._replace(rev=rev, frozen=frozen)
+
+
+class RepositoryCannotBeUpdatedError(RuntimeError):
+ pass
+
+
+def _check_hooks_still_exist_at_rev(
+ repo_config: Dict[str, Any],
+ info: RevInfo,
+ store: Store,
+) -> None:
+ try:
+ path = store.clone(repo_config['repo'], info.rev)
+ manifest = load_manifest(os.path.join(path, C.MANIFEST_FILE))
+ except InvalidManifestError as e:
+ raise RepositoryCannotBeUpdatedError(str(e))
+
+ # See if any of our hooks were deleted with the new commits
+ hooks = {hook['id'] for hook in repo_config['hooks']}
+ hooks_missing = hooks - {hook['id'] for hook in manifest}
+ if hooks_missing:
+ raise RepositoryCannotBeUpdatedError(
+ f'Cannot update because the tip of HEAD is missing these hooks:\n'
+ f'{", ".join(sorted(hooks_missing))}',
+ )
+
+
+REV_LINE_RE = re.compile(r'^(\s+)rev:(\s*)([^\s#]+)(.*)(\r?\n)$', re.DOTALL)
+
+
+def _original_lines(
+ path: str,
+ rev_infos: List[Optional[RevInfo]],
+ retry: bool = False,
+) -> Tuple[List[str], List[int]]:
+ """detect `rev:` lines or reformat the file"""
+ with open(path) as f:
+ original = f.read()
+
+ lines = original.splitlines(True)
+ idxs = [i for i, line in enumerate(lines) if REV_LINE_RE.match(line)]
+ if len(idxs) == len(rev_infos):
+ return lines, idxs
+ elif retry:
+ raise AssertionError('could not find rev lines')
+ else:
+ with open(path, 'w') as f:
+ f.write(yaml_dump(yaml_load(original)))
+ return _original_lines(path, rev_infos, retry=True)
+
+
+def _write_new_config(path: str, rev_infos: List[Optional[RevInfo]]) -> None:
+ lines, idxs = _original_lines(path, rev_infos)
+
+ for idx, rev_info in zip(idxs, rev_infos):
+ if rev_info is None:
+ continue
+ match = REV_LINE_RE.match(lines[idx])
+ assert match is not None
+ new_rev_s = yaml_dump({'rev': rev_info.rev})
+ new_rev = new_rev_s.split(':', 1)[1].strip()
+ if rev_info.frozen is not None:
+ comment = f' # frozen: {rev_info.frozen}'
+ elif match[4].strip().startswith('# frozen:'):
+ comment = ''
+ else:
+ comment = match[4]
+ lines[idx] = f'{match[1]}rev:{match[2]}{new_rev}{comment}{match[5]}'
+
+ with open(path, 'w') as f:
+ f.write(''.join(lines))
+
+
+def autoupdate(
+ config_file: str,
+ store: Store,
+ tags_only: bool,
+ freeze: bool,
+ repos: Sequence[str] = (),
+) -> int:
+ """Auto-update the pre-commit config to the latest versions of repos."""
+ migrate_config(config_file, quiet=True)
+ retv = 0
+ rev_infos: List[Optional[RevInfo]] = []
+ changed = False
+
+ config = load_config(config_file)
+ for repo_config in config['repos']:
+ if repo_config['repo'] in {LOCAL, META}:
+ continue
+
+ info = RevInfo.from_config(repo_config)
+ if repos and info.repo not in repos:
+ rev_infos.append(None)
+ continue
+
+ output.write(f'Updating {info.repo} ... ')
+ new_info = info.update(tags_only=tags_only, freeze=freeze)
+ try:
+ _check_hooks_still_exist_at_rev(repo_config, new_info, store)
+ except RepositoryCannotBeUpdatedError as error:
+ output.write_line(error.args[0])
+ rev_infos.append(None)
+ retv = 1
+ continue
+
+ if new_info.rev != info.rev:
+ changed = True
+ if new_info.frozen:
+ updated_to = f'{new_info.frozen} (frozen)'
+ else:
+ updated_to = new_info.rev
+ msg = f'updating {info.rev} -> {updated_to}.'
+ output.write_line(msg)
+ rev_infos.append(new_info)
+ else:
+ output.write_line('already up to date.')
+ rev_infos.append(None)
+
+ if changed:
+ _write_new_config(config_file, rev_infos)
+
+ return retv
diff --git a/pre_commit/commands/clean.py b/pre_commit/commands/clean.py
new file mode 100644
index 0000000..2be6c16
--- /dev/null
+++ b/pre_commit/commands/clean.py
@@ -0,0 +1,14 @@
+import os.path
+
+from pre_commit import output
+from pre_commit.store import Store
+from pre_commit.util import rmtree
+
+
+def clean(store: Store) -> int:
+ legacy_path = os.path.expanduser('~/.pre-commit')
+ for directory in (store.directory, legacy_path):
+ if os.path.exists(directory):
+ rmtree(directory)
+ output.write_line(f'Cleaned {directory}.')
+ return 0
diff --git a/pre_commit/commands/gc.py b/pre_commit/commands/gc.py
new file mode 100644
index 0000000..7f6d311
--- /dev/null
+++ b/pre_commit/commands/gc.py
@@ -0,0 +1,90 @@
+import os.path
+from typing import Any
+from typing import Dict
+from typing import Set
+from typing import Tuple
+
+import pre_commit.constants as C
+from pre_commit import output
+from pre_commit.clientlib import InvalidConfigError
+from pre_commit.clientlib import InvalidManifestError
+from pre_commit.clientlib import load_config
+from pre_commit.clientlib import load_manifest
+from pre_commit.clientlib import LOCAL
+from pre_commit.clientlib import META
+from pre_commit.store import Store
+
+
+def _mark_used_repos(
+ store: Store,
+ all_repos: Dict[Tuple[str, str], str],
+ unused_repos: Set[Tuple[str, str]],
+ repo: Dict[str, Any],
+) -> None:
+ if repo['repo'] == META:
+ return
+ elif repo['repo'] == LOCAL:
+ for hook in repo['hooks']:
+ deps = hook.get('additional_dependencies')
+ unused_repos.discard((
+ store.db_repo_name(repo['repo'], deps), C.LOCAL_REPO_VERSION,
+ ))
+ else:
+ key = (repo['repo'], repo['rev'])
+ path = all_repos.get(key)
+ # can't inspect manifest if it isn't cloned
+ if path is None:
+ return
+
+ try:
+ manifest = load_manifest(os.path.join(path, C.MANIFEST_FILE))
+ except InvalidManifestError:
+ return
+ else:
+ unused_repos.discard(key)
+ by_id = {hook['id']: hook for hook in manifest}
+
+ for hook in repo['hooks']:
+ if hook['id'] not in by_id:
+ continue
+
+ deps = hook.get(
+ 'additional_dependencies',
+ by_id[hook['id']]['additional_dependencies'],
+ )
+ unused_repos.discard((
+ store.db_repo_name(repo['repo'], deps), repo['rev'],
+ ))
+
+
+def _gc_repos(store: Store) -> int:
+ configs = store.select_all_configs()
+ repos = store.select_all_repos()
+
+ # delete config paths which do not exist
+ dead_configs = [p for p in configs if not os.path.exists(p)]
+ live_configs = [p for p in configs if os.path.exists(p)]
+
+ all_repos = {(repo, ref): path for repo, ref, path in repos}
+ unused_repos = set(all_repos)
+ for config_path in live_configs:
+ try:
+ config = load_config(config_path)
+ except InvalidConfigError:
+ dead_configs.append(config_path)
+ continue
+ else:
+ for repo in config['repos']:
+ _mark_used_repos(store, all_repos, unused_repos, repo)
+
+ store.delete_configs(dead_configs)
+ for db_repo_name, ref in unused_repos:
+ store.delete_repo(db_repo_name, ref, all_repos[(db_repo_name, ref)])
+ return len(unused_repos)
+
+
+def gc(store: Store) -> int:
+ with store.exclusive_lock():
+ repos_removed = _gc_repos(store)
+ output.write_line(f'{repos_removed} repo(s) removed.')
+ return 0
diff --git a/pre_commit/commands/hook_impl.py b/pre_commit/commands/hook_impl.py
new file mode 100644
index 0000000..5ff4555
--- /dev/null
+++ b/pre_commit/commands/hook_impl.py
@@ -0,0 +1,187 @@
+import argparse
+import os.path
+import subprocess
+import sys
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+
+from pre_commit.commands.run import run
+from pre_commit.envcontext import envcontext
+from pre_commit.parse_shebang import normalize_cmd
+from pre_commit.store import Store
+
+Z40 = '0' * 40
+
+
+def _run_legacy(
+ hook_type: str,
+ hook_dir: str,
+ args: Sequence[str],
+) -> Tuple[int, bytes]:
+ if os.environ.get('PRE_COMMIT_RUNNING_LEGACY'):
+ raise SystemExit(
+ f"bug: pre-commit's script is installed in migration mode\n"
+ f'run `pre-commit install -f --hook-type {hook_type}` to fix '
+ f'this\n\n'
+ f'Please report this bug at '
+ f'https://github.com/pre-commit/pre-commit/issues',
+ )
+
+ if hook_type == 'pre-push':
+ stdin = sys.stdin.buffer.read()
+ else:
+ stdin = b''
+
+ # not running in legacy mode
+ legacy_hook = os.path.join(hook_dir, f'{hook_type}.legacy')
+ if not os.access(legacy_hook, os.X_OK):
+ return 0, stdin
+
+ with envcontext((('PRE_COMMIT_RUNNING_LEGACY', '1'),)):
+ cmd = normalize_cmd((legacy_hook, *args))
+ return subprocess.run(cmd, input=stdin).returncode, stdin
+
+
+def _validate_config(
+ retv: int,
+ config: str,
+ skip_on_missing_config: bool,
+) -> None:
+ if not os.path.isfile(config):
+ if skip_on_missing_config or os.getenv('PRE_COMMIT_ALLOW_NO_CONFIG'):
+ print(f'`{config}` config file not found. Skipping `pre-commit`.')
+ raise SystemExit(retv)
+ else:
+ print(
+ f'No {config} file was found\n'
+ f'- To temporarily silence this, run '
+ f'`PRE_COMMIT_ALLOW_NO_CONFIG=1 git ...`\n'
+ f'- To permanently silence this, install pre-commit with the '
+ f'--allow-missing-config option\n'
+ f'- To uninstall pre-commit run `pre-commit uninstall`',
+ )
+ raise SystemExit(1)
+
+
+def _ns(
+ hook_type: str,
+ color: bool,
+ *,
+ all_files: bool = False,
+ from_ref: Optional[str] = None,
+ to_ref: Optional[str] = None,
+ remote_name: Optional[str] = None,
+ remote_url: Optional[str] = None,
+ commit_msg_filename: Optional[str] = None,
+ checkout_type: Optional[str] = None,
+) -> argparse.Namespace:
+ return argparse.Namespace(
+ color=color,
+ hook_stage=hook_type.replace('pre-', ''),
+ from_ref=from_ref,
+ to_ref=to_ref,
+ remote_name=remote_name,
+ remote_url=remote_url,
+ commit_msg_filename=commit_msg_filename,
+ all_files=all_files,
+ checkout_type=checkout_type,
+ files=(),
+ hook=None,
+ verbose=False,
+ show_diff_on_failure=False,
+ )
+
+
+def _rev_exists(rev: str) -> bool:
+ return not subprocess.call(('git', 'rev-list', '--quiet', rev))
+
+
+def _pre_push_ns(
+ color: bool,
+ args: Sequence[str],
+ stdin: bytes,
+) -> Optional[argparse.Namespace]:
+ remote_name = args[0]
+ remote_url = args[1]
+
+ for line in stdin.decode().splitlines():
+ _, local_sha, _, remote_sha = line.split()
+ if local_sha == Z40:
+ continue
+ elif remote_sha != Z40 and _rev_exists(remote_sha):
+ return _ns(
+ 'pre-push', color,
+ from_ref=remote_sha, to_ref=local_sha,
+ remote_name=remote_name, remote_url=remote_url,
+ )
+ else:
+ # ancestors not found in remote
+ ancestors = subprocess.check_output((
+ 'git', 'rev-list', local_sha, '--topo-order', '--reverse',
+ '--not', f'--remotes={remote_name}',
+ )).decode().strip()
+ if not ancestors:
+ continue
+ else:
+ first_ancestor = ancestors.splitlines()[0]
+ cmd = ('git', 'rev-list', '--max-parents=0', local_sha)
+ roots = set(subprocess.check_output(cmd).decode().splitlines())
+ if first_ancestor in roots:
+ # pushing the whole tree including root commit
+ return _ns(
+ 'pre-push', color,
+ all_files=True,
+ remote_name=remote_name, remote_url=remote_url,
+ )
+ else:
+ rev_cmd = ('git', 'rev-parse', f'{first_ancestor}^')
+ source = subprocess.check_output(rev_cmd).decode().strip()
+ return _ns(
+ 'pre-push', color,
+ from_ref=source, to_ref=local_sha,
+ remote_name=remote_name, remote_url=remote_url,
+ )
+
+ # nothing to push
+ return None
+
+
+def _run_ns(
+ hook_type: str,
+ color: bool,
+ args: Sequence[str],
+ stdin: bytes,
+) -> Optional[argparse.Namespace]:
+ if hook_type == 'pre-push':
+ return _pre_push_ns(color, args, stdin)
+ elif hook_type in {'prepare-commit-msg', 'commit-msg'}:
+ return _ns(hook_type, color, commit_msg_filename=args[0])
+ elif hook_type in {'pre-merge-commit', 'pre-commit'}:
+ return _ns(hook_type, color)
+ elif hook_type == 'post-checkout':
+ return _ns(
+ hook_type, color,
+ from_ref=args[0], to_ref=args[1], checkout_type=args[2],
+ )
+ else:
+ raise AssertionError(f'unexpected hook type: {hook_type}')
+
+
+def hook_impl(
+ store: Store,
+ *,
+ config: str,
+ color: bool,
+ hook_type: str,
+ hook_dir: str,
+ skip_on_missing_config: bool,
+ args: Sequence[str],
+) -> int:
+ retv, stdin = _run_legacy(hook_type, hook_dir, args)
+ _validate_config(retv, config, skip_on_missing_config)
+ ns = _run_ns(hook_type, color, args, stdin)
+ if ns is None:
+ return retv
+ else:
+ return retv | run(config, store, ns)
diff --git a/pre_commit/commands/init_templatedir.py b/pre_commit/commands/init_templatedir.py
new file mode 100644
index 0000000..f676fb1
--- /dev/null
+++ b/pre_commit/commands/init_templatedir.py
@@ -0,0 +1,33 @@
+import logging
+import os.path
+from typing import Sequence
+
+from pre_commit.commands.install_uninstall import install
+from pre_commit.store import Store
+from pre_commit.util import CalledProcessError
+from pre_commit.util import cmd_output
+
+logger = logging.getLogger('pre_commit')
+
+
+def init_templatedir(
+ config_file: str,
+ store: Store,
+ directory: str,
+ hook_types: Sequence[str],
+) -> int:
+ install(
+ config_file, store, hook_types=hook_types,
+ overwrite=True, skip_on_missing_config=True, git_dir=directory,
+ )
+ try:
+ _, out, _ = cmd_output('git', 'config', 'init.templateDir')
+ except CalledProcessError:
+ configured_path = None
+ else:
+ configured_path = os.path.realpath(os.path.expanduser(out.strip()))
+ dest = os.path.realpath(directory)
+ if configured_path != dest:
+ logger.warning('`init.templateDir` not set to the target directory')
+ logger.warning(f'maybe `git config --global init.templateDir {dest}`?')
+ return 0
diff --git a/pre_commit/commands/install_uninstall.py b/pre_commit/commands/install_uninstall.py
new file mode 100644
index 0000000..c8b7633
--- /dev/null
+++ b/pre_commit/commands/install_uninstall.py
@@ -0,0 +1,175 @@
+import itertools
+import logging
+import os.path
+import shutil
+import sys
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+
+from pre_commit import git
+from pre_commit import output
+from pre_commit.clientlib import load_config
+from pre_commit.repository import all_hooks
+from pre_commit.repository import install_hook_envs
+from pre_commit.store import Store
+from pre_commit.util import make_executable
+from pre_commit.util import resource_text
+
+
+logger = logging.getLogger(__name__)
+
+# This is used to identify the hook file we install
+PRIOR_HASHES = (
+ '4d9958c90bc262f47553e2c073f14cfe',
+ 'd8ee923c46731b42cd95cc869add4062',
+ '49fd668cb42069aa1b6048464be5d395',
+ '79f09a650522a87b0da915d0d983b2de',
+ 'e358c9dae00eac5d06b38dfdb1e33a8c',
+)
+CURRENT_HASH = '138fd403232d2ddd5efb44317e38bf03'
+TEMPLATE_START = '# start templated\n'
+TEMPLATE_END = '# end templated\n'
+# Homebrew/homebrew-core#35825: be more timid about appropriate `PATH`
+# #1312 os.defpath is too restrictive on BSD
+POSIX_SEARCH_PATH = ('/usr/local/bin', '/usr/bin', '/bin')
+SYS_EXE = os.path.basename(os.path.realpath(sys.executable))
+
+
+def _hook_paths(
+ hook_type: str,
+ git_dir: Optional[str] = None,
+) -> Tuple[str, str]:
+ git_dir = git_dir if git_dir is not None else git.get_git_dir()
+ pth = os.path.join(git_dir, 'hooks', hook_type)
+ return pth, f'{pth}.legacy'
+
+
+def is_our_script(filename: str) -> bool:
+ if not os.path.exists(filename): # pragma: win32 no cover (symlink)
+ return False
+ with open(filename) as f:
+ contents = f.read()
+ return any(h in contents for h in (CURRENT_HASH,) + PRIOR_HASHES)
+
+
+def shebang() -> str:
+ if sys.platform == 'win32':
+ py = SYS_EXE
+ else:
+ exe_choices = [
+ f'python{sys.version_info[0]}.{sys.version_info[1]}',
+ f'python{sys.version_info[0]}',
+ ]
+ # avoid searching for bare `python` as it's likely to be python 2
+ if SYS_EXE != 'python':
+ exe_choices.append(SYS_EXE)
+ for path, exe in itertools.product(POSIX_SEARCH_PATH, exe_choices):
+ if os.access(os.path.join(path, exe), os.X_OK):
+ py = exe
+ break
+ else:
+ py = SYS_EXE
+ return f'#!/usr/bin/env {py}'
+
+
+def _install_hook_script(
+ config_file: str,
+ hook_type: str,
+ overwrite: bool = False,
+ skip_on_missing_config: bool = False,
+ git_dir: Optional[str] = None,
+) -> None:
+ hook_path, legacy_path = _hook_paths(hook_type, git_dir=git_dir)
+
+ os.makedirs(os.path.dirname(hook_path), exist_ok=True)
+
+ # If we have an existing hook, move it to pre-commit.legacy
+ if os.path.lexists(hook_path) and not is_our_script(hook_path):
+ shutil.move(hook_path, legacy_path)
+
+ # If we specify overwrite, we simply delete the legacy file
+ if overwrite and os.path.exists(legacy_path):
+ os.remove(legacy_path)
+ elif os.path.exists(legacy_path):
+ output.write_line(
+ f'Running in migration mode with existing hooks at {legacy_path}\n'
+ f'Use -f to use only pre-commit.',
+ )
+
+ args = ['hook-impl', f'--config={config_file}', f'--hook-type={hook_type}']
+ if skip_on_missing_config:
+ args.append('--skip-on-missing-config')
+ params = {'INSTALL_PYTHON': sys.executable, 'ARGS': args}
+
+ with open(hook_path, 'w') as hook_file:
+ contents = resource_text('hook-tmpl')
+ before, rest = contents.split(TEMPLATE_START)
+ to_template, after = rest.split(TEMPLATE_END)
+
+ before = before.replace('#!/usr/bin/env python3', shebang())
+
+ hook_file.write(before + TEMPLATE_START)
+ for line in to_template.splitlines():
+ var = line.split()[0]
+ hook_file.write(f'{var} = {params[var]!r}\n')
+ hook_file.write(TEMPLATE_END + after)
+ make_executable(hook_path)
+
+ output.write_line(f'pre-commit installed at {hook_path}')
+
+
+def install(
+ config_file: str,
+ store: Store,
+ hook_types: Sequence[str],
+ overwrite: bool = False,
+ hooks: bool = False,
+ skip_on_missing_config: bool = False,
+ git_dir: Optional[str] = None,
+) -> int:
+ if git_dir is None and git.has_core_hookpaths_set():
+ logger.error(
+ 'Cowardly refusing to install hooks with `core.hooksPath` set.\n'
+ 'hint: `git config --unset-all core.hooksPath`',
+ )
+ return 1
+
+ for hook_type in hook_types:
+ _install_hook_script(
+ config_file, hook_type,
+ overwrite=overwrite,
+ skip_on_missing_config=skip_on_missing_config,
+ git_dir=git_dir,
+ )
+
+ if hooks:
+ install_hooks(config_file, store)
+
+ return 0
+
+
+def install_hooks(config_file: str, store: Store) -> int:
+ install_hook_envs(all_hooks(load_config(config_file), store), store)
+ return 0
+
+
+def _uninstall_hook_script(hook_type: str) -> None:
+ hook_path, legacy_path = _hook_paths(hook_type)
+
+ # If our file doesn't exist or it isn't ours, gtfo.
+ if not os.path.exists(hook_path) or not is_our_script(hook_path):
+ return
+
+ os.remove(hook_path)
+ output.write_line(f'{hook_type} uninstalled')
+
+ if os.path.exists(legacy_path):
+ os.rename(legacy_path, hook_path)
+ output.write_line(f'Restored previous hooks to {hook_path}')
+
+
+def uninstall(hook_types: Sequence[str]) -> int:
+ for hook_type in hook_types:
+ _uninstall_hook_script(hook_type)
+ return 0
diff --git a/pre_commit/commands/migrate_config.py b/pre_commit/commands/migrate_config.py
new file mode 100644
index 0000000..d83b8e9
--- /dev/null
+++ b/pre_commit/commands/migrate_config.py
@@ -0,0 +1,59 @@
+import re
+
+import yaml
+
+from pre_commit.util import yaml_load
+
+
+def _indent(s: str) -> str:
+ lines = s.splitlines(True)
+ return ''.join(' ' * 4 + line if line.strip() else line for line in lines)
+
+
+def _is_header_line(line: str) -> bool:
+ return line.startswith(('#', '---')) or not line.strip()
+
+
+def _migrate_map(contents: str) -> str:
+ # Find the first non-header line
+ lines = contents.splitlines(True)
+ i = 0
+ # Only loop on non empty configuration file
+ while i < len(lines) and _is_header_line(lines[i]):
+ i += 1
+
+ header = ''.join(lines[:i])
+ rest = ''.join(lines[i:])
+
+ if isinstance(yaml_load(contents), list):
+ # If they are using the "default" flow style of yaml, this operation
+ # will yield a valid configuration
+ try:
+ trial_contents = f'{header}repos:\n{rest}'
+ yaml_load(trial_contents)
+ contents = trial_contents
+ except yaml.YAMLError:
+ contents = f'{header}repos:\n{_indent(rest)}'
+
+ return contents
+
+
+def _migrate_sha_to_rev(contents: str) -> str:
+ return re.sub(r'(\n\s+)sha:', r'\1rev:', contents)
+
+
+def migrate_config(config_file: str, quiet: bool = False) -> int:
+ with open(config_file) as f:
+ orig_contents = contents = f.read()
+
+ contents = _migrate_map(contents)
+ contents = _migrate_sha_to_rev(contents)
+
+ if contents != orig_contents:
+ with open(config_file, 'w') as f:
+ f.write(contents)
+
+ print('Configuration has been migrated.')
+ elif not quiet:
+ print('Configuration is already migrated.')
+ return 0
diff --git a/pre_commit/commands/run.py b/pre_commit/commands/run.py
new file mode 100644
index 0000000..2f74578
--- /dev/null
+++ b/pre_commit/commands/run.py
@@ -0,0 +1,360 @@
+import argparse
+import contextlib
+import functools
+import logging
+import os
+import re
+import subprocess
+import time
+from typing import Any
+from typing import Collection
+from typing import Dict
+from typing import List
+from typing import Sequence
+from typing import Set
+from typing import Tuple
+
+from identify.identify import tags_from_path
+
+from pre_commit import color
+from pre_commit import git
+from pre_commit import output
+from pre_commit.clientlib import load_config
+from pre_commit.hook import Hook
+from pre_commit.languages.all import languages
+from pre_commit.repository import all_hooks
+from pre_commit.repository import install_hook_envs
+from pre_commit.staged_files_only import staged_files_only
+from pre_commit.store import Store
+from pre_commit.util import cmd_output_b
+from pre_commit.util import EnvironT
+
+
+logger = logging.getLogger('pre_commit')
+
+
+def _start_msg(*, start: str, cols: int, end_len: int) -> str:
+ dots = '.' * (cols - len(start) - end_len - 1)
+ return f'{start}{dots}'
+
+
+def _full_msg(
+ *,
+ start: str,
+ cols: int,
+ end_msg: str,
+ end_color: str,
+ use_color: bool,
+ postfix: str = '',
+) -> str:
+ dots = '.' * (cols - len(start) - len(postfix) - len(end_msg) - 1)
+ end = color.format_color(end_msg, end_color, use_color)
+ return f'{start}{dots}{postfix}{end}\n'
+
+
+def filter_by_include_exclude(
+ names: Collection[str],
+ include: str,
+ exclude: str,
+) -> List[str]:
+ include_re, exclude_re = re.compile(include), re.compile(exclude)
+ return [
+ filename for filename in names
+ if include_re.search(filename)
+ if not exclude_re.search(filename)
+ ]
+
+
+class Classifier:
+ def __init__(self, filenames: Sequence[str]) -> None:
+ # on windows we normalize all filenames to use forward slashes
+ # this makes it easier to filter using the `files:` regex
+ # this also makes improperly quoted shell-based hooks work better
+ # see #1173
+ if os.altsep == '/' and os.sep == '\\':
+ filenames = [f.replace(os.sep, os.altsep) for f in filenames]
+ self.filenames = [f for f in filenames if os.path.lexists(f)]
+
+ @functools.lru_cache(maxsize=None)
+ def _types_for_file(self, filename: str) -> Set[str]:
+ return tags_from_path(filename)
+
+ def by_types(
+ self,
+ names: Sequence[str],
+ types: Collection[str],
+ exclude_types: Collection[str],
+ ) -> List[str]:
+ types, exclude_types = frozenset(types), frozenset(exclude_types)
+ ret = []
+ for filename in names:
+ tags = self._types_for_file(filename)
+ if tags >= types and not tags & exclude_types:
+ ret.append(filename)
+ return ret
+
+ def filenames_for_hook(self, hook: Hook) -> Tuple[str, ...]:
+ names = self.filenames
+ names = filter_by_include_exclude(names, hook.files, hook.exclude)
+ names = self.by_types(names, hook.types, hook.exclude_types)
+ return tuple(names)
+
+
+def _get_skips(environ: EnvironT) -> Set[str]:
+ skips = environ.get('SKIP', '')
+ return {skip.strip() for skip in skips.split(',') if skip.strip()}
+
+
+SKIPPED = 'Skipped'
+NO_FILES = '(no files to check)'
+
+
+def _subtle_line(s: str, use_color: bool) -> None:
+ output.write_line(color.format_color(s, color.SUBTLE, use_color))
+
+
+def _run_single_hook(
+ classifier: Classifier,
+ hook: Hook,
+ skips: Set[str],
+ cols: int,
+ verbose: bool,
+ use_color: bool,
+) -> bool:
+ filenames = classifier.filenames_for_hook(hook)
+
+ if hook.id in skips or hook.alias in skips:
+ output.write(
+ _full_msg(
+ start=hook.name,
+ end_msg=SKIPPED,
+ end_color=color.YELLOW,
+ use_color=use_color,
+ cols=cols,
+ ),
+ )
+ duration = None
+ retcode = 0
+ files_modified = False
+ out = b''
+ elif not filenames and not hook.always_run:
+ output.write(
+ _full_msg(
+ start=hook.name,
+ postfix=NO_FILES,
+ end_msg=SKIPPED,
+ end_color=color.TURQUOISE,
+ use_color=use_color,
+ cols=cols,
+ ),
+ )
+ duration = None
+ retcode = 0
+ files_modified = False
+ out = b''
+ else:
+ # print hook and dots first in case the hook takes a while to run
+ output.write(_start_msg(start=hook.name, end_len=6, cols=cols))
+
+ diff_cmd = ('git', 'diff', '--no-ext-diff')
+ diff_before = cmd_output_b(*diff_cmd, retcode=None)
+ if not hook.pass_filenames:
+ filenames = ()
+ time_before = time.time()
+ language = languages[hook.language]
+ retcode, out = language.run_hook(hook, filenames, use_color)
+ duration = round(time.time() - time_before, 2) or 0
+ diff_after = cmd_output_b(*diff_cmd, retcode=None)
+
+ # if the hook makes changes, fail the commit
+ files_modified = diff_before != diff_after
+
+ if retcode or files_modified:
+ print_color = color.RED
+ status = 'Failed'
+ else:
+ print_color = color.GREEN
+ status = 'Passed'
+
+ output.write_line(color.format_color(status, print_color, use_color))
+
+ if verbose or hook.verbose or retcode or files_modified:
+ _subtle_line(f'- hook id: {hook.id}', use_color)
+
+ if (verbose or hook.verbose) and duration is not None:
+ _subtle_line(f'- duration: {duration}s', use_color)
+
+ if retcode:
+ _subtle_line(f'- exit code: {retcode}', use_color)
+
+ # Print a message if failing due to file modifications
+ if files_modified:
+ _subtle_line('- files were modified by this hook', use_color)
+
+ if out.strip():
+ output.write_line()
+ output.write_line_b(out.strip(), logfile_name=hook.log_file)
+ output.write_line()
+
+ return files_modified or bool(retcode)
+
+
+def _compute_cols(hooks: Sequence[Hook]) -> int:
+ """Compute the number of columns to display hook messages. The widest
+ that will be displayed is in the no files skipped case:
+
+ Hook name...(no files to check) Skipped
+ """
+ if hooks:
+ name_len = max(len(hook.name) for hook in hooks)
+ else:
+ name_len = 0
+
+ cols = name_len + 3 + len(NO_FILES) + 1 + len(SKIPPED)
+ return max(cols, 80)
+
+
+def _all_filenames(args: argparse.Namespace) -> Collection[str]:
+ if args.hook_stage == 'post-checkout': # no files for post-checkout
+ return ()
+ elif args.hook_stage in {'prepare-commit-msg', 'commit-msg'}:
+ return (args.commit_msg_filename,)
+ elif args.from_ref and args.to_ref:
+ return git.get_changed_files(args.from_ref, args.to_ref)
+ elif args.files:
+ return args.files
+ elif args.all_files:
+ return git.get_all_files()
+ elif git.is_in_merge_conflict():
+ return git.get_conflicted_files()
+ else:
+ return git.get_staged_files()
+
+
+def _run_hooks(
+ config: Dict[str, Any],
+ hooks: Sequence[Hook],
+ args: argparse.Namespace,
+ environ: EnvironT,
+) -> int:
+ """Actually run the hooks."""
+ skips = _get_skips(environ)
+ cols = _compute_cols(hooks)
+ filenames = filter_by_include_exclude(
+ _all_filenames(args), config['files'], config['exclude'],
+ )
+ classifier = Classifier(filenames)
+ retval = 0
+ for hook in hooks:
+ retval |= _run_single_hook(
+ classifier, hook, skips, cols,
+ verbose=args.verbose, use_color=args.color,
+ )
+ if retval and config['fail_fast']:
+ break
+ if retval and args.show_diff_on_failure and git.has_diff():
+ if args.all_files:
+ output.write_line(
+ 'pre-commit hook(s) made changes.\n'
+ 'If you are seeing this message in CI, '
+ 'reproduce locally with: `pre-commit run --all-files`.\n'
+ 'To run `pre-commit` as part of git workflow, use '
+ '`pre-commit install`.',
+ )
+ output.write_line('All changes made by hooks:')
+ # args.color is a boolean.
+ # See user_color function in color.py
+ git_color_opt = 'always' if args.color else 'never'
+ subprocess.call((
+ 'git', '--no-pager', 'diff', '--no-ext-diff',
+ f'--color={git_color_opt}',
+ ))
+
+ return retval
+
+
+def _has_unmerged_paths() -> bool:
+ _, stdout, _ = cmd_output_b('git', 'ls-files', '--unmerged')
+ return bool(stdout.strip())
+
+
+def _has_unstaged_config(config_file: str) -> bool:
+ retcode, _, _ = cmd_output_b(
+ 'git', 'diff', '--no-ext-diff', '--exit-code', config_file,
+ retcode=None,
+ )
+ # be explicit, other git errors don't mean it has an unstaged config.
+ return retcode == 1
+
+
+def run(
+ config_file: str,
+ store: Store,
+ args: argparse.Namespace,
+ environ: EnvironT = os.environ,
+) -> int:
+ stash = not args.all_files and not args.files
+
+ # Check if we have unresolved merge conflict files and fail fast.
+ if _has_unmerged_paths():
+ logger.error('Unmerged files. Resolve before committing.')
+ return 1
+ if bool(args.from_ref) != bool(args.to_ref):
+ logger.error('Specify both --from-ref and --to-ref.')
+ return 1
+ if stash and _has_unstaged_config(config_file):
+ logger.error(
+ f'Your pre-commit configuration is unstaged.\n'
+ f'`git add {config_file}` to fix this.',
+ )
+ return 1
+ if (
+ args.hook_stage in {'prepare-commit-msg', 'commit-msg'} and
+ not args.commit_msg_filename
+ ):
+ logger.error(
+ f'`--commit-msg-filename` is required for '
+ f'`--hook-stage {args.hook_stage}`',
+ )
+ return 1
+
+ # Expose from-ref / to-ref as environment variables for hooks to consume
+ if args.from_ref and args.to_ref:
+ # legacy names
+ environ['PRE_COMMIT_ORIGIN'] = args.from_ref
+ environ['PRE_COMMIT_SOURCE'] = args.to_ref
+ # new names
+ environ['PRE_COMMIT_FROM_REF'] = args.from_ref
+ environ['PRE_COMMIT_TO_REF'] = args.to_ref
+
+ if args.remote_name and args.remote_url:
+ environ['PRE_COMMIT_REMOTE_NAME'] = args.remote_name
+ environ['PRE_COMMIT_REMOTE_URL'] = args.remote_url
+
+ if args.checkout_type:
+ environ['PRE_COMMIT_CHECKOUT_TYPE'] = args.checkout_type
+
+ with contextlib.ExitStack() as exit_stack:
+ if stash:
+ exit_stack.enter_context(staged_files_only(store.directory))
+
+ config = load_config(config_file)
+ hooks = [
+ hook
+ for hook in all_hooks(config, store)
+ if not args.hook or hook.id == args.hook or hook.alias == args.hook
+ if args.hook_stage in hook.stages
+ ]
+
+ if args.hook and not hooks:
+ output.write_line(
+ f'No hook with id `{args.hook}` in stage `{args.hook_stage}`',
+ )
+ return 1
+
+ install_hook_envs(hooks, store)
+
+ return _run_hooks(config, hooks, args, environ)
+
+ # https://github.com/python/mypy/issues/7726
+ raise AssertionError('unreachable')
diff --git a/pre_commit/commands/sample_config.py b/pre_commit/commands/sample_config.py
new file mode 100644
index 0000000..d435faa
--- /dev/null
+++ b/pre_commit/commands/sample_config.py
@@ -0,0 +1,21 @@
+# TODO: maybe `git ls-remote git://github.com/pre-commit/pre-commit-hooks` to
+# determine the latest revision? This adds ~200ms from my tests (and is
+# significantly faster than https:// or http://). For now, periodically
+# manually updating the revision is fine.
+SAMPLE_CONFIG = '''\
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v2.4.0
+ hooks:
+ - id: trailing-whitespace
+ - id: end-of-file-fixer
+ - id: check-yaml
+ - id: check-added-large-files
+'''
+
+
+def sample_config() -> int:
+ print(SAMPLE_CONFIG, end='')
+ return 0
diff --git a/pre_commit/commands/try_repo.py b/pre_commit/commands/try_repo.py
new file mode 100644
index 0000000..4aee209
--- /dev/null
+++ b/pre_commit/commands/try_repo.py
@@ -0,0 +1,77 @@
+import argparse
+import logging
+import os.path
+from typing import Optional
+from typing import Tuple
+
+import pre_commit.constants as C
+from pre_commit import git
+from pre_commit import output
+from pre_commit.clientlib import load_manifest
+from pre_commit.commands.run import run
+from pre_commit.store import Store
+from pre_commit.util import cmd_output_b
+from pre_commit.util import tmpdir
+from pre_commit.util import yaml_dump
+from pre_commit.xargs import xargs
+
+logger = logging.getLogger(__name__)
+
+
+def _repo_ref(tmpdir: str, repo: str, ref: Optional[str]) -> Tuple[str, str]:
+ # if `ref` is explicitly passed, use it
+ if ref is not None:
+ return repo, ref
+
+ ref = git.head_rev(repo)
+ # if it exists on disk, we'll try and clone it with the local changes
+ if os.path.exists(repo) and git.has_diff('HEAD', repo=repo):
+ logger.warning('Creating temporary repo with uncommitted changes...')
+
+ shadow = os.path.join(tmpdir, 'shadow-repo')
+ cmd_output_b('git', 'clone', repo, shadow)
+ cmd_output_b('git', 'checkout', ref, '-b', '_pc_tmp', cwd=shadow)
+
+ idx = git.git_path('index', repo=shadow)
+ objs = git.git_path('objects', repo=shadow)
+ env = dict(os.environ, GIT_INDEX_FILE=idx, GIT_OBJECT_DIRECTORY=objs)
+
+ staged_files = git.get_staged_files(cwd=repo)
+ if staged_files:
+ xargs(('git', 'add', '--'), staged_files, cwd=repo, env=env)
+
+ cmd_output_b('git', 'add', '-u', cwd=repo, env=env)
+ git.commit(repo=shadow)
+
+ return shadow, git.head_rev(shadow)
+ else:
+ return repo, ref
+
+
+def try_repo(args: argparse.Namespace) -> int:
+ with tmpdir() as tempdir:
+ repo, ref = _repo_ref(tempdir, args.repo, args.ref)
+
+ store = Store(tempdir)
+ if args.hook:
+ hooks = [{'id': args.hook}]
+ else:
+ repo_path = store.clone(repo, ref)
+ manifest = load_manifest(os.path.join(repo_path, C.MANIFEST_FILE))
+ manifest = sorted(manifest, key=lambda hook: hook['id'])
+ hooks = [{'id': hook['id']} for hook in manifest]
+
+ config = {'repos': [{'repo': repo, 'rev': ref, 'hooks': hooks}]}
+ config_s = yaml_dump(config)
+
+ config_filename = os.path.join(tempdir, C.CONFIG_FILE)
+ with open(config_filename, 'w') as cfg:
+ cfg.write(config_s)
+
+ output.write_line('=' * 79)
+ output.write_line('Using config:')
+ output.write_line('=' * 79)
+ output.write(config_s)
+ output.write_line('=' * 79)
+
+ return run(config_filename, store, args)
diff --git a/pre_commit/constants.py b/pre_commit/constants.py
new file mode 100644
index 0000000..e2b8e3a
--- /dev/null
+++ b/pre_commit/constants.py
@@ -0,0 +1,24 @@
+import sys
+
+if sys.version_info < (3, 8): # pragma: no cover (<PY38)
+ import importlib_metadata
+else: # pragma: no cover (PY38+)
+ import importlib.metadata as importlib_metadata
+
+CONFIG_FILE = '.pre-commit-config.yaml'
+MANIFEST_FILE = '.pre-commit-hooks.yaml'
+
+# Bump when installation changes in a backwards / forwards incompatible way
+INSTALLED_STATE_VERSION = '1'
+# Bump when modifying `empty_template`
+LOCAL_REPO_VERSION = '1'
+
+VERSION = importlib_metadata.version('pre_commit')
+
+# `manual` is not invoked by any installed git hook. See #719
+STAGES = (
+ 'commit', 'merge-commit', 'prepare-commit-msg', 'commit-msg', 'manual',
+ 'post-checkout', 'push',
+)
+
+DEFAULT = 'default'
diff --git a/pre_commit/envcontext.py b/pre_commit/envcontext.py
new file mode 100644
index 0000000..16d3d15
--- /dev/null
+++ b/pre_commit/envcontext.py
@@ -0,0 +1,67 @@
+import contextlib
+import enum
+import os
+from typing import Generator
+from typing import NamedTuple
+from typing import Optional
+from typing import Tuple
+from typing import Union
+
+from pre_commit.util import EnvironT
+
+
+class _Unset(enum.Enum):
+ UNSET = 1
+
+
+UNSET = _Unset.UNSET
+
+
+class Var(NamedTuple):
+ name: str
+ default: str = ''
+
+
+SubstitutionT = Tuple[Union[str, Var], ...]
+ValueT = Union[str, _Unset, SubstitutionT]
+PatchesT = Tuple[Tuple[str, ValueT], ...]
+
+
+def format_env(parts: SubstitutionT, env: EnvironT) -> str:
+ return ''.join(
+ env.get(part.name, part.default) if isinstance(part, Var) else part
+ for part in parts
+ )
+
+
+@contextlib.contextmanager
+def envcontext(
+ patch: PatchesT,
+ _env: Optional[EnvironT] = None,
+) -> Generator[None, None, None]:
+ """In this context, `os.environ` is modified according to `patch`.
+
+ `patch` is an iterable of 2-tuples (key, value):
+ `key`: string
+ `value`:
+ - string: `environ[key] == value` inside the context.
+ - UNSET: `key not in environ` inside the context.
+ - template: A template is a tuple of strings and Var which will be
+ replaced with the previous environment
+ """
+ env = os.environ if _env is None else _env
+ before = env.copy()
+
+ for k, v in patch:
+ if v is UNSET:
+ env.pop(k, None)
+ elif isinstance(v, tuple):
+ env[k] = format_env(v, before)
+ else:
+ env[k] = v
+
+ try:
+ yield
+ finally:
+ env.clear()
+ env.update(before)
diff --git a/pre_commit/error_handler.py b/pre_commit/error_handler.py
new file mode 100644
index 0000000..b2321ae
--- /dev/null
+++ b/pre_commit/error_handler.py
@@ -0,0 +1,64 @@
+import contextlib
+import functools
+import os.path
+import sys
+import traceback
+from typing import Generator
+
+import pre_commit.constants as C
+from pre_commit import output
+from pre_commit.store import Store
+from pre_commit.util import force_bytes
+
+
+class FatalError(RuntimeError):
+ pass
+
+
+def _log_and_exit(msg: str, exc: BaseException, formatted: str) -> None:
+ error_msg = f'{msg}: {type(exc).__name__}: '.encode() + force_bytes(exc)
+ output.write_line_b(error_msg)
+ log_path = os.path.join(Store().directory, 'pre-commit.log')
+ output.write_line(f'Check the log at {log_path}')
+
+ with open(log_path, 'wb') as log:
+ _log_line = functools.partial(output.write_line, stream=log)
+ _log_line_b = functools.partial(output.write_line_b, stream=log)
+
+ _log_line('### version information')
+ _log_line()
+ _log_line('```')
+ _log_line(f'pre-commit version: {C.VERSION}')
+ _log_line('sys.version:')
+ for line in sys.version.splitlines():
+ _log_line(f' {line}')
+ _log_line(f'sys.executable: {sys.executable}')
+ _log_line(f'os.name: {os.name}')
+ _log_line(f'sys.platform: {sys.platform}')
+ _log_line('```')
+ _log_line()
+
+ _log_line('### error information')
+ _log_line()
+ _log_line('```')
+ _log_line_b(error_msg)
+ _log_line('```')
+ _log_line()
+ _log_line('```')
+ _log_line(formatted)
+ _log_line('```')
+ raise SystemExit(1)
+
+
+@contextlib.contextmanager
+def error_handler() -> Generator[None, None, None]:
+ try:
+ yield
+ except (Exception, KeyboardInterrupt) as e:
+ if isinstance(e, FatalError):
+ msg = 'An error has occurred'
+ elif isinstance(e, KeyboardInterrupt):
+ msg = 'Interrupted (^C)'
+ else:
+ msg = 'An unexpected error has occurred'
+ _log_and_exit(msg, e, traceback.format_exc())
diff --git a/pre_commit/file_lock.py b/pre_commit/file_lock.py
new file mode 100644
index 0000000..ff0dc5e
--- /dev/null
+++ b/pre_commit/file_lock.py
@@ -0,0 +1,76 @@
+import contextlib
+import errno
+import os
+from typing import Callable
+from typing import Generator
+
+
+if os.name == 'nt': # pragma: no cover (windows)
+ import msvcrt
+
+ # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/locking
+
+ # on windows we lock "regions" of files, we don't care about the actual
+ # byte region so we'll just pick *some* number here.
+ _region = 0xffff
+
+ @contextlib.contextmanager
+ def _locked(
+ fileno: int,
+ blocked_cb: Callable[[], None],
+ ) -> Generator[None, None, None]:
+ try:
+ # TODO: https://github.com/python/typeshed/pull/3607
+ msvcrt.locking(fileno, msvcrt.LK_NBLCK, _region) # type: ignore
+ except OSError:
+ blocked_cb()
+ while True:
+ try:
+ # TODO: https://github.com/python/typeshed/pull/3607
+ msvcrt.locking(fileno, msvcrt.LK_LOCK, _region) # type: ignore # noqa: E501
+ except OSError as e:
+ # Locking violation. Returned when the _LK_LOCK or _LK_RLCK
+ # flag is specified and the file cannot be locked after 10
+ # attempts.
+ if e.errno != errno.EDEADLOCK:
+ raise
+ else:
+ break
+
+ try:
+ yield
+ finally:
+ # From cursory testing, it seems to get unlocked when the file is
+ # closed so this may not be necessary.
+ # The documentation however states:
+ # "Regions should be locked only briefly and should be unlocked
+ # before closing a file or exiting the program."
+ # TODO: https://github.com/python/typeshed/pull/3607
+ msvcrt.locking(fileno, msvcrt.LK_UNLCK, _region) # type: ignore
+else: # pragma: win32 no cover
+ import fcntl
+
+ @contextlib.contextmanager
+ def _locked(
+ fileno: int,
+ blocked_cb: Callable[[], None],
+ ) -> Generator[None, None, None]:
+ try:
+ fcntl.flock(fileno, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except OSError: # pragma: no cover (tests are single-threaded)
+ blocked_cb()
+ fcntl.flock(fileno, fcntl.LOCK_EX)
+ try:
+ yield
+ finally:
+ fcntl.flock(fileno, fcntl.LOCK_UN)
+
+
+@contextlib.contextmanager
+def lock(
+ path: str,
+ blocked_cb: Callable[[], None],
+) -> Generator[None, None, None]:
+ with open(path, 'a+') as f:
+ with _locked(f.fileno(), blocked_cb):
+ yield
diff --git a/pre_commit/git.py b/pre_commit/git.py
new file mode 100644
index 0000000..7e757f2
--- /dev/null
+++ b/pre_commit/git.py
@@ -0,0 +1,196 @@
+import logging
+import os.path
+import sys
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Set
+
+from pre_commit.util import cmd_output
+from pre_commit.util import cmd_output_b
+from pre_commit.util import EnvironT
+
+
+logger = logging.getLogger(__name__)
+
+
+def zsplit(s: str) -> List[str]:
+ s = s.strip('\0')
+ if s:
+ return s.split('\0')
+ else:
+ return []
+
+
+def no_git_env(_env: Optional[EnvironT] = None) -> Dict[str, str]:
+ # Too many bugs dealing with environment variables and GIT:
+ # https://github.com/pre-commit/pre-commit/issues/300
+ # In git 2.6.3 (maybe others), git exports GIT_WORK_TREE while running
+ # pre-commit hooks
+ # In git 1.9.1 (maybe others), git exports GIT_DIR and GIT_INDEX_FILE
+ # while running pre-commit hooks in submodules.
+ # GIT_DIR: Causes git clone to clone wrong thing
+ # GIT_INDEX_FILE: Causes 'error invalid object ...' during commit
+ _env = _env if _env is not None else os.environ
+ return {
+ k: v for k, v in _env.items()
+ if not k.startswith('GIT_') or
+ k in {
+ 'GIT_EXEC_PATH', 'GIT_SSH', 'GIT_SSH_COMMAND', 'GIT_SSL_CAINFO',
+ 'GIT_SSL_NO_VERIFY',
+ }
+ }
+
+
+def get_root() -> str:
+ return cmd_output('git', 'rev-parse', '--show-toplevel')[1].strip()
+
+
+def get_git_dir(git_root: str = '.') -> str:
+ opts = ('--git-common-dir', '--git-dir')
+ _, out, _ = cmd_output('git', 'rev-parse', *opts, cwd=git_root)
+ for line, opt in zip(out.splitlines(), opts):
+ if line != opt: # pragma: no branch (git < 2.5)
+ return os.path.normpath(os.path.join(git_root, line))
+ else:
+ raise AssertionError('unreachable: no git dir')
+
+
+def get_remote_url(git_root: str) -> str:
+ _, out, _ = cmd_output('git', 'config', 'remote.origin.url', cwd=git_root)
+ return out.strip()
+
+
+def is_in_merge_conflict() -> bool:
+ git_dir = get_git_dir('.')
+ return (
+ os.path.exists(os.path.join(git_dir, 'MERGE_MSG')) and
+ os.path.exists(os.path.join(git_dir, 'MERGE_HEAD'))
+ )
+
+
+def parse_merge_msg_for_conflicts(merge_msg: bytes) -> List[str]:
+ # Conflicted files start with tabs
+ return [
+ line.lstrip(b'#').strip().decode()
+ for line in merge_msg.splitlines()
+ # '#\t' for git 2.4.1
+ if line.startswith((b'\t', b'#\t'))
+ ]
+
+
+def get_conflicted_files() -> Set[str]:
+ logger.info('Checking merge-conflict files only.')
+ # Need to get the conflicted files from the MERGE_MSG because they could
+ # have resolved the conflict by choosing one side or the other
+ with open(os.path.join(get_git_dir('.'), 'MERGE_MSG'), 'rb') as f:
+ merge_msg = f.read()
+ merge_conflict_filenames = parse_merge_msg_for_conflicts(merge_msg)
+
+ # This will get the rest of the changes made after the merge.
+ # If they resolved the merge conflict by choosing a mesh of both sides
+ # this will also include the conflicted files
+ tree_hash = cmd_output('git', 'write-tree')[1].strip()
+ merge_diff_filenames = zsplit(
+ cmd_output(
+ 'git', 'diff', '--name-only', '--no-ext-diff', '-z',
+ '-m', tree_hash, 'HEAD', 'MERGE_HEAD',
+ )[1],
+ )
+ return set(merge_conflict_filenames) | set(merge_diff_filenames)
+
+
+def get_staged_files(cwd: Optional[str] = None) -> List[str]:
+ return zsplit(
+ cmd_output(
+ 'git', 'diff', '--staged', '--name-only', '--no-ext-diff', '-z',
+ # Everything except for D
+ '--diff-filter=ACMRTUXB',
+ cwd=cwd,
+ )[1],
+ )
+
+
+def intent_to_add_files() -> List[str]:
+ _, stdout, _ = cmd_output('git', 'status', '--porcelain', '-z')
+ parts = list(reversed(zsplit(stdout)))
+ intent_to_add = []
+ while parts:
+ line = parts.pop()
+ status, filename = line[:3], line[3:]
+ if status[0] in {'C', 'R'}: # renames / moves have an additional arg
+ parts.pop()
+ if status[1] == 'A':
+ intent_to_add.append(filename)
+ return intent_to_add
+
+
+def get_all_files() -> List[str]:
+ return zsplit(cmd_output('git', 'ls-files', '-z')[1])
+
+
+def get_changed_files(old: str, new: str) -> List[str]:
+ return zsplit(
+ cmd_output(
+ 'git', 'diff', '--name-only', '--no-ext-diff', '-z',
+ f'{old}...{new}',
+ )[1],
+ )
+
+
+def head_rev(remote: str) -> str:
+ _, out, _ = cmd_output('git', 'ls-remote', '--exit-code', remote, 'HEAD')
+ return out.split()[0]
+
+
+def has_diff(*args: str, repo: str = '.') -> bool:
+ cmd = ('git', 'diff', '--quiet', '--no-ext-diff', *args)
+ return cmd_output_b(*cmd, cwd=repo, retcode=None)[0] == 1
+
+
+def has_core_hookpaths_set() -> bool:
+ _, out, _ = cmd_output_b('git', 'config', 'core.hooksPath', retcode=None)
+ return bool(out.strip())
+
+
+def init_repo(path: str, remote: str) -> None:
+ if os.path.isdir(remote):
+ remote = os.path.abspath(remote)
+
+ env = no_git_env()
+ cmd_output_b('git', 'init', path, env=env)
+ cmd_output_b('git', 'remote', 'add', 'origin', remote, cwd=path, env=env)
+
+
+def commit(repo: str = '.') -> None:
+ env = no_git_env()
+ name, email = 'pre-commit', 'asottile+pre-commit@umich.edu'
+ env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = name
+ env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = email
+ cmd = ('git', 'commit', '--no-edit', '--no-gpg-sign', '-n', '-minit')
+ cmd_output_b(*cmd, cwd=repo, env=env)
+
+
+def git_path(name: str, repo: str = '.') -> str:
+ _, out, _ = cmd_output('git', 'rev-parse', '--git-path', name, cwd=repo)
+ return os.path.join(repo, out.strip())
+
+
+def check_for_cygwin_mismatch() -> None:
+ """See https://github.com/pre-commit/pre-commit/issues/354"""
+ if sys.platform in ('cygwin', 'win32'): # pragma: no cover (windows)
+ is_cygwin_python = sys.platform == 'cygwin'
+ toplevel = cmd_output('git', 'rev-parse', '--show-toplevel')[1]
+ is_cygwin_git = toplevel.startswith('/')
+
+ if is_cygwin_python ^ is_cygwin_git:
+ exe_type = {True: '(cygwin)', False: '(windows)'}
+ logger.warn(
+ f'pre-commit has detected a mix of cygwin python / git\n'
+ f'This combination is not supported, it is likely you will '
+ f'receive an error later in the program.\n'
+ f'Make sure to use cygwin git+python while using cygwin\n'
+ f'These can be installed through the cygwin installer.\n'
+ f' - python {exe_type[is_cygwin_python]}\n'
+ f' - git {exe_type[is_cygwin_git]}\n',
+ )
diff --git a/pre_commit/hook.py b/pre_commit/hook.py
new file mode 100644
index 0000000..b65ac42
--- /dev/null
+++ b/pre_commit/hook.py
@@ -0,0 +1,63 @@
+import logging
+import shlex
+from typing import Any
+from typing import Dict
+from typing import NamedTuple
+from typing import Sequence
+from typing import Tuple
+
+from pre_commit.prefix import Prefix
+
+logger = logging.getLogger('pre_commit')
+
+
+class Hook(NamedTuple):
+ src: str
+ prefix: Prefix
+ id: str
+ name: str
+ entry: str
+ language: str
+ alias: str
+ files: str
+ exclude: str
+ types: Sequence[str]
+ exclude_types: Sequence[str]
+ additional_dependencies: Sequence[str]
+ args: Sequence[str]
+ always_run: bool
+ pass_filenames: bool
+ description: str
+ language_version: str
+ log_file: str
+ minimum_pre_commit_version: str
+ require_serial: bool
+ stages: Sequence[str]
+ verbose: bool
+
+ @property
+ def cmd(self) -> Tuple[str, ...]:
+ return (*shlex.split(self.entry), *self.args)
+
+ @property
+ def install_key(self) -> Tuple[Prefix, str, str, Tuple[str, ...]]:
+ return (
+ self.prefix,
+ self.language,
+ self.language_version,
+ tuple(self.additional_dependencies),
+ )
+
+ @classmethod
+ def create(cls, src: str, prefix: Prefix, dct: Dict[str, Any]) -> 'Hook':
+ # TODO: have cfgv do this (?)
+ extra_keys = set(dct) - _KEYS
+ if extra_keys:
+ logger.warning(
+ f'Unexpected key(s) present on {src} => {dct["id"]}: '
+ f'{", ".join(sorted(extra_keys))}',
+ )
+ return cls(src=src, prefix=prefix, **{k: dct[k] for k in _KEYS})
+
+
+_KEYS = frozenset(set(Hook._fields) - {'src', 'prefix'})
diff --git a/pre_commit/languages/__init__.py b/pre_commit/languages/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/pre_commit/languages/__init__.py
diff --git a/pre_commit/languages/all.py b/pre_commit/languages/all.py
new file mode 100644
index 0000000..8f4ffa8
--- /dev/null
+++ b/pre_commit/languages/all.py
@@ -0,0 +1,60 @@
+from typing import Callable
+from typing import NamedTuple
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+
+from pre_commit.hook import Hook
+from pre_commit.languages import conda
+from pre_commit.languages import docker
+from pre_commit.languages import docker_image
+from pre_commit.languages import fail
+from pre_commit.languages import golang
+from pre_commit.languages import node
+from pre_commit.languages import perl
+from pre_commit.languages import pygrep
+from pre_commit.languages import python
+from pre_commit.languages import python_venv
+from pre_commit.languages import ruby
+from pre_commit.languages import rust
+from pre_commit.languages import script
+from pre_commit.languages import swift
+from pre_commit.languages import system
+from pre_commit.prefix import Prefix
+
+
+class Language(NamedTuple):
+ name: str
+ # Use `None` for no installation / environment
+ ENVIRONMENT_DIR: Optional[str]
+ # return a value to replace `'default` for `language_version`
+ get_default_version: Callable[[], str]
+ # return whether the environment is healthy (or should be rebuilt)
+ healthy: Callable[[Prefix, str], bool]
+ # install a repository for the given language and language_version
+ install_environment: Callable[[Prefix, str, Sequence[str]], None]
+ # execute a hook and return the exit code and output
+ run_hook: 'Callable[[Hook, Sequence[str], bool], Tuple[int, bytes]]'
+
+
+# TODO: back to modules + Protocol: https://github.com/python/mypy/issues/5018
+languages = {
+ # BEGIN GENERATED (testing/gen-languages-all)
+ 'conda': Language(name='conda', ENVIRONMENT_DIR=conda.ENVIRONMENT_DIR, get_default_version=conda.get_default_version, healthy=conda.healthy, install_environment=conda.install_environment, run_hook=conda.run_hook), # noqa: E501
+ 'docker': Language(name='docker', ENVIRONMENT_DIR=docker.ENVIRONMENT_DIR, get_default_version=docker.get_default_version, healthy=docker.healthy, install_environment=docker.install_environment, run_hook=docker.run_hook), # noqa: E501
+ 'docker_image': Language(name='docker_image', ENVIRONMENT_DIR=docker_image.ENVIRONMENT_DIR, get_default_version=docker_image.get_default_version, healthy=docker_image.healthy, install_environment=docker_image.install_environment, run_hook=docker_image.run_hook), # noqa: E501
+ 'fail': Language(name='fail', ENVIRONMENT_DIR=fail.ENVIRONMENT_DIR, get_default_version=fail.get_default_version, healthy=fail.healthy, install_environment=fail.install_environment, run_hook=fail.run_hook), # noqa: E501
+ 'golang': Language(name='golang', ENVIRONMENT_DIR=golang.ENVIRONMENT_DIR, get_default_version=golang.get_default_version, healthy=golang.healthy, install_environment=golang.install_environment, run_hook=golang.run_hook), # noqa: E501
+ 'node': Language(name='node', ENVIRONMENT_DIR=node.ENVIRONMENT_DIR, get_default_version=node.get_default_version, healthy=node.healthy, install_environment=node.install_environment, run_hook=node.run_hook), # noqa: E501
+ 'perl': Language(name='perl', ENVIRONMENT_DIR=perl.ENVIRONMENT_DIR, get_default_version=perl.get_default_version, healthy=perl.healthy, install_environment=perl.install_environment, run_hook=perl.run_hook), # noqa: E501
+ 'pygrep': Language(name='pygrep', ENVIRONMENT_DIR=pygrep.ENVIRONMENT_DIR, get_default_version=pygrep.get_default_version, healthy=pygrep.healthy, install_environment=pygrep.install_environment, run_hook=pygrep.run_hook), # noqa: E501
+ 'python': Language(name='python', ENVIRONMENT_DIR=python.ENVIRONMENT_DIR, get_default_version=python.get_default_version, healthy=python.healthy, install_environment=python.install_environment, run_hook=python.run_hook), # noqa: E501
+ 'python_venv': Language(name='python_venv', ENVIRONMENT_DIR=python_venv.ENVIRONMENT_DIR, get_default_version=python_venv.get_default_version, healthy=python_venv.healthy, install_environment=python_venv.install_environment, run_hook=python_venv.run_hook), # noqa: E501
+ 'ruby': Language(name='ruby', ENVIRONMENT_DIR=ruby.ENVIRONMENT_DIR, get_default_version=ruby.get_default_version, healthy=ruby.healthy, install_environment=ruby.install_environment, run_hook=ruby.run_hook), # noqa: E501
+ 'rust': Language(name='rust', ENVIRONMENT_DIR=rust.ENVIRONMENT_DIR, get_default_version=rust.get_default_version, healthy=rust.healthy, install_environment=rust.install_environment, run_hook=rust.run_hook), # noqa: E501
+ 'script': Language(name='script', ENVIRONMENT_DIR=script.ENVIRONMENT_DIR, get_default_version=script.get_default_version, healthy=script.healthy, install_environment=script.install_environment, run_hook=script.run_hook), # noqa: E501
+ 'swift': Language(name='swift', ENVIRONMENT_DIR=swift.ENVIRONMENT_DIR, get_default_version=swift.get_default_version, healthy=swift.healthy, install_environment=swift.install_environment, run_hook=swift.run_hook), # noqa: E501
+ 'system': Language(name='system', ENVIRONMENT_DIR=system.ENVIRONMENT_DIR, get_default_version=system.get_default_version, healthy=system.healthy, install_environment=system.install_environment, run_hook=system.run_hook), # noqa: E501
+ # END GENERATED
+}
+all_languages = sorted(languages)
diff --git a/pre_commit/languages/conda.py b/pre_commit/languages/conda.py
new file mode 100644
index 0000000..071757a
--- /dev/null
+++ b/pre_commit/languages/conda.py
@@ -0,0 +1,84 @@
+import contextlib
+import os
+from typing import Generator
+from typing import Sequence
+from typing import Tuple
+
+from pre_commit.envcontext import envcontext
+from pre_commit.envcontext import PatchesT
+from pre_commit.envcontext import SubstitutionT
+from pre_commit.envcontext import UNSET
+from pre_commit.envcontext import Var
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+from pre_commit.prefix import Prefix
+from pre_commit.util import clean_path_on_failure
+from pre_commit.util import cmd_output_b
+
+ENVIRONMENT_DIR = 'conda'
+get_default_version = helpers.basic_get_default_version
+healthy = helpers.basic_healthy
+
+
+def get_env_patch(env: str) -> PatchesT:
+ # On non-windows systems executable live in $CONDA_PREFIX/bin, on Windows
+ # they can be in $CONDA_PREFIX/bin, $CONDA_PREFIX/Library/bin,
+ # $CONDA_PREFIX/Scripts and $CONDA_PREFIX. Whereas the latter only
+ # seems to be used for python.exe.
+ path: SubstitutionT = (os.path.join(env, 'bin'), os.pathsep, Var('PATH'))
+ if os.name == 'nt': # pragma: no cover (platform specific)
+ path = (env, os.pathsep, *path)
+ path = (os.path.join(env, 'Scripts'), os.pathsep, *path)
+ path = (os.path.join(env, 'Library', 'bin'), os.pathsep, *path)
+
+ return (
+ ('PYTHONHOME', UNSET),
+ ('VIRTUAL_ENV', UNSET),
+ ('CONDA_PREFIX', env),
+ ('PATH', path),
+ )
+
+
+@contextlib.contextmanager
+def in_env(
+ prefix: Prefix,
+ language_version: str,
+) -> Generator[None, None, None]:
+ directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
+ envdir = prefix.path(directory)
+ with envcontext(get_env_patch(envdir)):
+ yield
+
+
+def install_environment(
+ prefix: Prefix,
+ version: str,
+ additional_dependencies: Sequence[str],
+) -> None:
+ helpers.assert_version_default('conda', version)
+ directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
+
+ env_dir = prefix.path(directory)
+ with clean_path_on_failure(env_dir):
+ cmd_output_b(
+ 'conda', 'env', 'create', '-p', env_dir, '--file',
+ 'environment.yml', cwd=prefix.prefix_dir,
+ )
+ if additional_dependencies:
+ cmd_output_b(
+ 'conda', 'install', '-p', env_dir, *additional_dependencies,
+ cwd=prefix.prefix_dir,
+ )
+
+
+def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+) -> Tuple[int, bytes]:
+ # TODO: Some rare commands need to be run using `conda run` but mostly we
+ # can run them withot which is much quicker and produces a better
+ # output.
+ # cmd = ('conda', 'run', '-p', env_dir) + hook.cmd
+ with in_env(hook.prefix, hook.language_version):
+ return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
diff --git a/pre_commit/languages/docker.py b/pre_commit/languages/docker.py
new file mode 100644
index 0000000..f449584
--- /dev/null
+++ b/pre_commit/languages/docker.py
@@ -0,0 +1,114 @@
+import hashlib
+import os
+from typing import Sequence
+from typing import Tuple
+
+import pre_commit.constants as C
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+from pre_commit.prefix import Prefix
+from pre_commit.util import CalledProcessError
+from pre_commit.util import clean_path_on_failure
+from pre_commit.util import cmd_output_b
+
+ENVIRONMENT_DIR = 'docker'
+PRE_COMMIT_LABEL = 'PRE_COMMIT'
+get_default_version = helpers.basic_get_default_version
+healthy = helpers.basic_healthy
+
+
+def md5(s: str) -> str: # pragma: win32 no cover
+ return hashlib.md5(s.encode()).hexdigest()
+
+
+def docker_tag(prefix: Prefix) -> str: # pragma: win32 no cover
+ md5sum = md5(os.path.basename(prefix.prefix_dir)).lower()
+ return f'pre-commit-{md5sum}'
+
+
+def docker_is_running() -> bool: # pragma: win32 no cover
+ try:
+ cmd_output_b('docker', 'ps')
+ except CalledProcessError:
+ return False
+ else:
+ return True
+
+
+def assert_docker_available() -> None: # pragma: win32 no cover
+ assert docker_is_running(), (
+ 'Docker is either not running or not configured in this environment'
+ )
+
+
+def build_docker_image(
+ prefix: Prefix,
+ *,
+ pull: bool,
+) -> None: # pragma: win32 no cover
+ cmd: Tuple[str, ...] = (
+ 'docker', 'build',
+ '--tag', docker_tag(prefix),
+ '--label', PRE_COMMIT_LABEL,
+ )
+ if pull:
+ cmd += ('--pull',)
+ # This must come last for old versions of docker. See #477
+ cmd += ('.',)
+ helpers.run_setup_cmd(prefix, cmd)
+
+
+def install_environment(
+ prefix: Prefix, version: str, additional_dependencies: Sequence[str],
+) -> None: # pragma: win32 no cover
+ helpers.assert_version_default('docker', version)
+ helpers.assert_no_additional_deps('docker', additional_dependencies)
+ assert_docker_available()
+
+ directory = prefix.path(
+ helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
+ )
+
+ # Docker doesn't really have relevant disk environment, but pre-commit
+ # still needs to cleanup its state files on failure
+ with clean_path_on_failure(directory):
+ build_docker_image(prefix, pull=True)
+ os.mkdir(directory)
+
+
+def get_docker_user() -> str: # pragma: win32 no cover
+ try:
+ return f'{os.getuid()}:{os.getgid()}'
+ except AttributeError:
+ return '1000:1000'
+
+
+def docker_cmd() -> Tuple[str, ...]: # pragma: win32 no cover
+ return (
+ 'docker', 'run',
+ '--rm',
+ '-u', get_docker_user(),
+ # https://docs.docker.com/engine/reference/commandline/run/#mount-volumes-from-container-volumes-from
+ # The `Z` option tells Docker to label the content with a private
+ # unshared label. Only the current container can use a private volume.
+ '-v', f'{os.getcwd()}:/src:rw,Z',
+ '--workdir', '/src',
+ )
+
+
+def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+) -> Tuple[int, bytes]: # pragma: win32 no cover
+ assert_docker_available()
+ # Rebuild the docker image in case it has gone missing, as many people do
+ # automated cleanup of docker images.
+ build_docker_image(hook.prefix, pull=False)
+
+ hook_cmd = hook.cmd
+ entry_exe, cmd_rest = hook.cmd[0], hook_cmd[1:]
+
+ entry_tag = ('--entrypoint', entry_exe, docker_tag(hook.prefix))
+ cmd = docker_cmd() + entry_tag + cmd_rest
+ return helpers.run_xargs(hook, cmd, file_args, color=color)
diff --git a/pre_commit/languages/docker_image.py b/pre_commit/languages/docker_image.py
new file mode 100644
index 0000000..0c51df6
--- /dev/null
+++ b/pre_commit/languages/docker_image.py
@@ -0,0 +1,22 @@
+from typing import Sequence
+from typing import Tuple
+
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+from pre_commit.languages.docker import assert_docker_available
+from pre_commit.languages.docker import docker_cmd
+
+ENVIRONMENT_DIR = None
+get_default_version = helpers.basic_get_default_version
+healthy = helpers.basic_healthy
+install_environment = helpers.no_install
+
+
+def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+) -> Tuple[int, bytes]: # pragma: win32 no cover
+ assert_docker_available()
+ cmd = docker_cmd() + hook.cmd
+ return helpers.run_xargs(hook, cmd, file_args, color=color)
diff --git a/pre_commit/languages/fail.py b/pre_commit/languages/fail.py
new file mode 100644
index 0000000..d2b02d2
--- /dev/null
+++ b/pre_commit/languages/fail.py
@@ -0,0 +1,20 @@
+from typing import Sequence
+from typing import Tuple
+
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+
+ENVIRONMENT_DIR = None
+get_default_version = helpers.basic_get_default_version
+healthy = helpers.basic_healthy
+install_environment = helpers.no_install
+
+
+def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+) -> Tuple[int, bytes]:
+ out = f'{hook.entry}\n\n'.encode()
+ out += b'\n'.join(f.encode() for f in file_args) + b'\n'
+ return 1, out
diff --git a/pre_commit/languages/golang.py b/pre_commit/languages/golang.py
new file mode 100644
index 0000000..91ade1e
--- /dev/null
+++ b/pre_commit/languages/golang.py
@@ -0,0 +1,97 @@
+import contextlib
+import os.path
+import sys
+from typing import Generator
+from typing import Sequence
+from typing import Tuple
+
+import pre_commit.constants as C
+from pre_commit import git
+from pre_commit.envcontext import envcontext
+from pre_commit.envcontext import PatchesT
+from pre_commit.envcontext import Var
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+from pre_commit.prefix import Prefix
+from pre_commit.util import clean_path_on_failure
+from pre_commit.util import cmd_output
+from pre_commit.util import cmd_output_b
+from pre_commit.util import rmtree
+
+ENVIRONMENT_DIR = 'golangenv'
+get_default_version = helpers.basic_get_default_version
+healthy = helpers.basic_healthy
+
+
+def get_env_patch(venv: str) -> PatchesT:
+ return (
+ ('PATH', (os.path.join(venv, 'bin'), os.pathsep, Var('PATH'))),
+ )
+
+
+@contextlib.contextmanager
+def in_env(prefix: Prefix) -> Generator[None, None, None]:
+ envdir = prefix.path(
+ helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
+ )
+ with envcontext(get_env_patch(envdir)):
+ yield
+
+
+def guess_go_dir(remote_url: str) -> str:
+ if remote_url.endswith('.git'):
+ remote_url = remote_url[:-1 * len('.git')]
+ looks_like_url = (
+ not remote_url.startswith('file://') and
+ ('//' in remote_url or '@' in remote_url)
+ )
+ remote_url = remote_url.replace(':', '/')
+ if looks_like_url:
+ _, _, remote_url = remote_url.rpartition('//')
+ _, _, remote_url = remote_url.rpartition('@')
+ return remote_url
+ else:
+ return 'unknown_src_dir'
+
+
+def install_environment(
+ prefix: Prefix,
+ version: str,
+ additional_dependencies: Sequence[str],
+) -> None:
+ helpers.assert_version_default('golang', version)
+ directory = prefix.path(
+ helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
+ )
+
+ with clean_path_on_failure(directory):
+ remote = git.get_remote_url(prefix.prefix_dir)
+ repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote))
+
+ # Clone into the goenv we'll create
+ helpers.run_setup_cmd(prefix, ('git', 'clone', '.', repo_src_dir))
+
+ if sys.platform == 'cygwin': # pragma: no cover
+ _, gopath, _ = cmd_output('cygpath', '-w', directory)
+ gopath = gopath.strip()
+ else:
+ gopath = directory
+ env = dict(os.environ, GOPATH=gopath)
+ env.pop('GOBIN', None)
+ cmd_output_b('go', 'get', './...', cwd=repo_src_dir, env=env)
+ for dependency in additional_dependencies:
+ cmd_output_b('go', 'get', dependency, cwd=repo_src_dir, env=env)
+ # Same some disk space, we don't need these after installation
+ rmtree(prefix.path(directory, 'src'))
+ pkgdir = prefix.path(directory, 'pkg')
+ if os.path.exists(pkgdir): # pragma: no cover (go<1.10)
+ rmtree(pkgdir)
+
+
+def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+) -> Tuple[int, bytes]:
+ with in_env(hook.prefix):
+ return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
diff --git a/pre_commit/languages/helpers.py b/pre_commit/languages/helpers.py
new file mode 100644
index 0000000..b5c95e5
--- /dev/null
+++ b/pre_commit/languages/helpers.py
@@ -0,0 +1,109 @@
+import multiprocessing
+import os
+import random
+from typing import Any
+from typing import List
+from typing import Optional
+from typing import overload
+from typing import Sequence
+from typing import Tuple
+from typing import TYPE_CHECKING
+
+import pre_commit.constants as C
+from pre_commit.hook import Hook
+from pre_commit.prefix import Prefix
+from pre_commit.util import cmd_output_b
+from pre_commit.xargs import xargs
+
+if TYPE_CHECKING:
+ from typing import NoReturn
+
+FIXED_RANDOM_SEED = 1542676186
+
+
+def run_setup_cmd(prefix: Prefix, cmd: Tuple[str, ...]) -> None:
+ cmd_output_b(*cmd, cwd=prefix.prefix_dir)
+
+
+@overload
+def environment_dir(d: None, language_version: str) -> None: ...
+@overload
+def environment_dir(d: str, language_version: str) -> str: ...
+
+
+def environment_dir(d: Optional[str], language_version: str) -> Optional[str]:
+ if d is None:
+ return None
+ else:
+ return f'{d}-{language_version}'
+
+
+def assert_version_default(binary: str, version: str) -> None:
+ if version != C.DEFAULT:
+ raise AssertionError(
+ f'For now, pre-commit requires system-installed {binary}',
+ )
+
+
+def assert_no_additional_deps(
+ lang: str,
+ additional_deps: Sequence[str],
+) -> None:
+ if additional_deps:
+ raise AssertionError(
+ f'For now, pre-commit does not support '
+ f'additional_dependencies for {lang}',
+ )
+
+
+def basic_get_default_version() -> str:
+ return C.DEFAULT
+
+
+def basic_healthy(prefix: Prefix, language_version: str) -> bool:
+ return True
+
+
+def no_install(
+ prefix: Prefix,
+ version: str,
+ additional_dependencies: Sequence[str],
+) -> 'NoReturn':
+ raise AssertionError('This type is not installable')
+
+
+def target_concurrency(hook: Hook) -> int:
+ if hook.require_serial or 'PRE_COMMIT_NO_CONCURRENCY' in os.environ:
+ return 1
+ else:
+ # Travis appears to have a bunch of CPUs, but we can't use them all.
+ if 'TRAVIS' in os.environ:
+ return 2
+ else:
+ try:
+ return multiprocessing.cpu_count()
+ except NotImplementedError:
+ return 1
+
+
+def _shuffled(seq: Sequence[str]) -> List[str]:
+ """Deterministically shuffle"""
+ fixed_random = random.Random()
+ fixed_random.seed(FIXED_RANDOM_SEED, version=1)
+
+ seq = list(seq)
+ random.shuffle(seq, random=fixed_random.random)
+ return seq
+
+
+def run_xargs(
+ hook: Hook,
+ cmd: Tuple[str, ...],
+ file_args: Sequence[str],
+ **kwargs: Any,
+) -> Tuple[int, bytes]:
+ # Shuffle the files so that they more evenly fill out the xargs partitions,
+ # but do it deterministically in case a hook cares about ordering.
+ file_args = _shuffled(file_args)
+ kwargs['target_concurrency'] = target_concurrency(hook)
+ return xargs(cmd, file_args, **kwargs)
diff --git a/pre_commit/languages/node.py b/pre_commit/languages/node.py
new file mode 100644
index 0000000..79ff807
--- /dev/null
+++ b/pre_commit/languages/node.py
@@ -0,0 +1,93 @@
+import contextlib
+import os
+import sys
+from typing import Generator
+from typing import Sequence
+from typing import Tuple
+
+import pre_commit.constants as C
+from pre_commit.envcontext import envcontext
+from pre_commit.envcontext import PatchesT
+from pre_commit.envcontext import Var
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+from pre_commit.languages.python import bin_dir
+from pre_commit.prefix import Prefix
+from pre_commit.util import clean_path_on_failure
+from pre_commit.util import cmd_output
+from pre_commit.util import cmd_output_b
+
+ENVIRONMENT_DIR = 'node_env'
+get_default_version = helpers.basic_get_default_version
+healthy = helpers.basic_healthy
+
+
+def _envdir(prefix: Prefix, version: str) -> str:
+ directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
+ return prefix.path(directory)
+
+
+def get_env_patch(venv: str) -> PatchesT:
+ if sys.platform == 'cygwin': # pragma: no cover
+ _, win_venv, _ = cmd_output('cygpath', '-w', venv)
+ install_prefix = fr'{win_venv.strip()}\bin'
+ lib_dir = 'lib'
+ elif sys.platform == 'win32': # pragma: no cover
+ install_prefix = bin_dir(venv)
+ lib_dir = 'Scripts'
+ else: # pragma: win32 no cover
+ install_prefix = venv
+ lib_dir = 'lib'
+ return (
+ ('NODE_VIRTUAL_ENV', venv),
+ ('NPM_CONFIG_PREFIX', install_prefix),
+ ('npm_config_prefix', install_prefix),
+ ('NODE_PATH', os.path.join(venv, lib_dir, 'node_modules')),
+ ('PATH', (bin_dir(venv), os.pathsep, Var('PATH'))),
+ )
+
+
+@contextlib.contextmanager
+def in_env(
+ prefix: Prefix,
+ language_version: str,
+) -> Generator[None, None, None]:
+ with envcontext(get_env_patch(_envdir(prefix, language_version))):
+ yield
+
+
+def install_environment(
+ prefix: Prefix, version: str, additional_dependencies: Sequence[str],
+) -> None:
+ additional_dependencies = tuple(additional_dependencies)
+ assert prefix.exists('package.json')
+ envdir = _envdir(prefix, version)
+
+ # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx?f=255&MSPPError=-2147217396#maxpath
+ if sys.platform == 'win32': # pragma: no cover
+ envdir = f'\\\\?\\{os.path.normpath(envdir)}'
+ with clean_path_on_failure(envdir):
+ cmd = [
+ sys.executable, '-mnodeenv', '--prebuilt', '--clean-src', envdir,
+ ]
+ if version != C.DEFAULT:
+ cmd.extend(['-n', version])
+ cmd_output_b(*cmd)
+
+ with in_env(prefix, version):
+ # https://npm.community/t/npm-install-g-git-vs-git-clone-cd-npm-install-g/5449
+ # install as if we installed from git
+ helpers.run_setup_cmd(prefix, ('npm', 'install'))
+ helpers.run_setup_cmd(
+ prefix,
+ ('npm', 'install', '-g', '.', *additional_dependencies),
+ )
+
+
+def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+) -> Tuple[int, bytes]:
+ with in_env(hook.prefix, hook.language_version):
+ return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
diff --git a/pre_commit/languages/perl.py b/pre_commit/languages/perl.py
new file mode 100644
index 0000000..bbf5504
--- /dev/null
+++ b/pre_commit/languages/perl.py
@@ -0,0 +1,67 @@
+import contextlib
+import os
+import shlex
+from typing import Generator
+from typing import Sequence
+from typing import Tuple
+
+from pre_commit.envcontext import envcontext
+from pre_commit.envcontext import PatchesT
+from pre_commit.envcontext import Var
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+from pre_commit.prefix import Prefix
+from pre_commit.util import clean_path_on_failure
+
+ENVIRONMENT_DIR = 'perl_env'
+get_default_version = helpers.basic_get_default_version
+healthy = helpers.basic_healthy
+
+
+def _envdir(prefix: Prefix, version: str) -> str:
+ directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
+ return prefix.path(directory)
+
+
+def get_env_patch(venv: str) -> PatchesT:
+ return (
+ ('PATH', (os.path.join(venv, 'bin'), os.pathsep, Var('PATH'))),
+ ('PERL5LIB', os.path.join(venv, 'lib', 'perl5')),
+ ('PERL_MB_OPT', f'--install_base {shlex.quote(venv)}'),
+ (
+ 'PERL_MM_OPT', (
+ f'INSTALL_BASE={shlex.quote(venv)} '
+ f'INSTALLSITEMAN1DIR=none INSTALLSITEMAN3DIR=none'
+ ),
+ ),
+ )
+
+
+@contextlib.contextmanager
+def in_env(
+ prefix: Prefix,
+ language_version: str,
+) -> Generator[None, None, None]:
+ with envcontext(get_env_patch(_envdir(prefix, language_version))):
+ yield
+
+
+def install_environment(
+ prefix: Prefix, version: str, additional_dependencies: Sequence[str],
+) -> None:
+ helpers.assert_version_default('perl', version)
+
+ with clean_path_on_failure(_envdir(prefix, version)):
+ with in_env(prefix, version):
+ helpers.run_setup_cmd(
+ prefix, ('cpan', '-T', '.', *additional_dependencies),
+ )
+
+
+def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+) -> Tuple[int, bytes]:
+ with in_env(hook.prefix, hook.language_version):
+ return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
diff --git a/pre_commit/languages/pygrep.py b/pre_commit/languages/pygrep.py
new file mode 100644
index 0000000..40adba0
--- /dev/null
+++ b/pre_commit/languages/pygrep.py
@@ -0,0 +1,87 @@
+import argparse
+import re
+import sys
+from typing import Optional
+from typing import Pattern
+from typing import Sequence
+from typing import Tuple
+
+from pre_commit import output
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+from pre_commit.xargs import xargs
+
+ENVIRONMENT_DIR = None
+get_default_version = helpers.basic_get_default_version
+healthy = helpers.basic_healthy
+install_environment = helpers.no_install
+
+
+def _process_filename_by_line(pattern: Pattern[bytes], filename: str) -> int:
+ retv = 0
+ with open(filename, 'rb') as f:
+ for line_no, line in enumerate(f, start=1):
+ if pattern.search(line):
+ retv = 1
+ output.write(f'{filename}:{line_no}:')
+ output.write_line_b(line.rstrip(b'\r\n'))
+ return retv
+
+
+def _process_filename_at_once(pattern: Pattern[bytes], filename: str) -> int:
+ retv = 0
+ with open(filename, 'rb') as f:
+ contents = f.read()
+ match = pattern.search(contents)
+ if match:
+ retv = 1
+ line_no = contents[:match.start()].count(b'\n')
+ output.write(f'{filename}:{line_no + 1}:')
+
+ matched_lines = match[0].split(b'\n')
+ matched_lines[0] = contents.split(b'\n')[line_no]
+
+ output.write_line_b(b'\n'.join(matched_lines))
+ return retv
+
+
+def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+) -> Tuple[int, bytes]:
+ exe = (sys.executable, '-m', __name__) + tuple(hook.args) + (hook.entry,)
+ return xargs(exe, file_args, color=color)
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+ parser = argparse.ArgumentParser(
+ description=(
+ 'grep-like finder using python regexes. Unlike grep, this tool '
+ 'returns nonzero when it finds a match and zero otherwise. The '
+ 'idea here being that matches are "problems".'
+ ),
+ )
+ parser.add_argument('-i', '--ignore-case', action='store_true')
+ parser.add_argument('--multiline', action='store_true')
+ parser.add_argument('pattern', help='python regex pattern.')
+ parser.add_argument('filenames', nargs='*')
+ args = parser.parse_args(argv)
+
+ flags = re.IGNORECASE if args.ignore_case else 0
+ if args.multiline:
+ flags |= re.MULTILINE | re.DOTALL
+
+ pattern = re.compile(args.pattern.encode(), flags)
+
+ retv = 0
+ for filename in args.filenames:
+ if args.multiline:
+ retv |= _process_filename_at_once(pattern, filename)
+ else:
+ retv |= _process_filename_by_line(pattern, filename)
+ return retv
+
+
+if __name__ == '__main__':
+ exit(main())
diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py
new file mode 100644
index 0000000..5073a8b
--- /dev/null
+++ b/pre_commit/languages/python.py
@@ -0,0 +1,210 @@
+import contextlib
+import functools
+import os
+import sys
+from typing import Callable
+from typing import ContextManager
+from typing import Generator
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+
+import pre_commit.constants as C
+from pre_commit.envcontext import envcontext
+from pre_commit.envcontext import PatchesT
+from pre_commit.envcontext import UNSET
+from pre_commit.envcontext import Var
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+from pre_commit.parse_shebang import find_executable
+from pre_commit.prefix import Prefix
+from pre_commit.util import CalledProcessError
+from pre_commit.util import clean_path_on_failure
+from pre_commit.util import cmd_output
+from pre_commit.util import cmd_output_b
+
+ENVIRONMENT_DIR = 'py_env'
+
+
+def bin_dir(venv: str) -> str:
+ """On windows there's a different directory for the virtualenv"""
+ bin_part = 'Scripts' if os.name == 'nt' else 'bin'
+ return os.path.join(venv, bin_part)
+
+
+def get_env_patch(venv: str) -> PatchesT:
+ return (
+ ('PYTHONHOME', UNSET),
+ ('VIRTUAL_ENV', venv),
+ ('PATH', (bin_dir(venv), os.pathsep, Var('PATH'))),
+ )
+
+
+def _find_by_py_launcher(
+ version: str,
+) -> Optional[str]: # pragma: no cover (windows only)
+ if version.startswith('python'):
+ num = version[len('python'):]
+ try:
+ cmd = ('py', f'-{num}', '-c', 'import sys; print(sys.executable)')
+ return cmd_output(*cmd)[1].strip()
+ except CalledProcessError:
+ pass
+ return None
+
+
+def _find_by_sys_executable() -> Optional[str]:
+ def _norm(path: str) -> Optional[str]:
+ _, exe = os.path.split(path.lower())
+ exe, _, _ = exe.partition('.exe')
+ if exe not in {'python', 'pythonw'} and find_executable(exe):
+ return exe
+ return None
+
+ # On linux, I see these common sys.executables:
+ #
+ # system `python`: /usr/bin/python -> python2.7
+ # system `python2`: /usr/bin/python2 -> python2.7
+ # virtualenv v: v/bin/python (will not return from this loop)
+ # virtualenv v -ppython2: v/bin/python -> python2
+ # virtualenv v -ppython2.7: v/bin/python -> python2.7
+ # virtualenv v -ppypy: v/bin/python -> v/bin/pypy
+ for path in (sys.executable, os.path.realpath(sys.executable)):
+ exe = _norm(path)
+ if exe:
+ return exe
+ return None
+
+
+@functools.lru_cache(maxsize=1)
+def get_default_version() -> str: # pragma: no cover (platform dependent)
+ # First attempt from `sys.executable` (or the realpath)
+ exe = _find_by_sys_executable()
+ if exe:
+ return exe
+
+ # Next try the `pythonX.X` executable
+ exe = f'python{sys.version_info[0]}.{sys.version_info[1]}'
+ if find_executable(exe):
+ return exe
+
+ if _find_by_py_launcher(exe):
+ return exe
+
+ # Give a best-effort try for windows
+ default_folder_name = exe.replace('.', '')
+ if os.path.exists(fr'C:\{default_folder_name}\python.exe'):
+ return exe
+
+ # We tried!
+ return C.DEFAULT
+
+
+def _sys_executable_matches(version: str) -> bool:
+ if version == 'python':
+ return True
+ elif not version.startswith('python'):
+ return False
+
+ try:
+ info = tuple(int(p) for p in version[len('python'):].split('.'))
+ except ValueError:
+ return False
+
+ return sys.version_info[:len(info)] == info
+
+
+def norm_version(version: str) -> str:
+ # first see if our current executable is appropriate
+ if _sys_executable_matches(version):
+ return sys.executable
+
+ if os.name == 'nt': # pragma: no cover (windows)
+ version_exec = _find_by_py_launcher(version)
+ if version_exec:
+ return version_exec
+
+ # Try looking up by name
+ version_exec = find_executable(version)
+ if version_exec and version_exec != version:
+ return version_exec
+
+ # If it is in the form pythonx.x search in the default
+ # place on windows
+ if version.startswith('python'):
+ default_folder_name = version.replace('.', '')
+ return fr'C:\{default_folder_name}\python.exe'
+
+ # Otherwise assume it is a path
+ return os.path.expanduser(version)
+
+
+def py_interface(
+ _dir: str,
+ _make_venv: Callable[[str, str], None],
+) -> Tuple[
+ Callable[[Prefix, str], ContextManager[None]],
+ Callable[[Prefix, str], bool],
+ Callable[[Hook, Sequence[str], bool], Tuple[int, bytes]],
+ Callable[[Prefix, str, Sequence[str]], None],
+]:
+ @contextlib.contextmanager
+ def in_env(
+ prefix: Prefix,
+ language_version: str,
+ ) -> Generator[None, None, None]:
+ envdir = prefix.path(helpers.environment_dir(_dir, language_version))
+ with envcontext(get_env_patch(envdir)):
+ yield
+
+ def healthy(prefix: Prefix, language_version: str) -> bool:
+ envdir = helpers.environment_dir(_dir, language_version)
+ exe_name = 'python.exe' if sys.platform == 'win32' else 'python'
+ py_exe = prefix.path(bin_dir(envdir), exe_name)
+ with in_env(prefix, language_version):
+ retcode, _, _ = cmd_output_b(
+ py_exe, '-c', 'import ctypes, datetime, io, os, ssl, weakref',
+ cwd='/',
+ retcode=None,
+ )
+ return retcode == 0
+
+ def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+ ) -> Tuple[int, bytes]:
+ with in_env(hook.prefix, hook.language_version):
+ return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
+
+ def install_environment(
+ prefix: Prefix,
+ version: str,
+ additional_dependencies: Sequence[str],
+ ) -> None:
+ additional_dependencies = tuple(additional_dependencies)
+ directory = helpers.environment_dir(_dir, version)
+
+ env_dir = prefix.path(directory)
+ with clean_path_on_failure(env_dir):
+ if version != C.DEFAULT:
+ python = norm_version(version)
+ else:
+ python = os.path.realpath(sys.executable)
+ _make_venv(env_dir, python)
+ with in_env(prefix, version):
+ helpers.run_setup_cmd(
+ prefix, ('pip', 'install', '.') + additional_dependencies,
+ )
+
+ return in_env, healthy, run_hook, install_environment
+
+
+def make_venv(envdir: str, python: str) -> None:
+ env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1')
+ cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python)
+ cmd_output_b(*cmd, env=env, cwd='/')
+
+
+_interface = py_interface(ENVIRONMENT_DIR, make_venv)
+in_env, healthy, run_hook, install_environment = _interface
diff --git a/pre_commit/languages/python_venv.py b/pre_commit/languages/python_venv.py
new file mode 100644
index 0000000..5404c8b
--- /dev/null
+++ b/pre_commit/languages/python_venv.py
@@ -0,0 +1,46 @@
+import os.path
+
+from pre_commit.languages import python
+from pre_commit.util import CalledProcessError
+from pre_commit.util import cmd_output
+from pre_commit.util import cmd_output_b
+
+ENVIRONMENT_DIR = 'py_venv'
+get_default_version = python.get_default_version
+
+
+def orig_py_exe(exe: str) -> str: # pragma: no cover (platform specific)
+ """A -mvenv virtualenv made from a -mvirtualenv virtualenv installs
+ packages to the incorrect location. Attempt to find the _original_ exe
+ and invoke `-mvenv` from there.
+
+ See:
+ - https://github.com/pre-commit/pre-commit/issues/755
+ - https://github.com/pypa/virtualenv/issues/1095
+ - https://bugs.python.org/issue30811
+ """
+ try:
+ prefix_script = 'import sys; print(sys.real_prefix)'
+ _, prefix, _ = cmd_output(exe, '-c', prefix_script)
+ prefix = prefix.strip()
+ except CalledProcessError:
+ # not created from -mvirtualenv
+ return exe
+
+ if os.name == 'nt':
+ expected = os.path.join(prefix, 'python.exe')
+ else:
+ expected = os.path.join(prefix, 'bin', os.path.basename(exe))
+
+ if os.path.exists(expected):
+ return expected
+ else:
+ return exe
+
+
+def make_venv(envdir: str, python: str) -> None:
+ cmd_output_b(orig_py_exe(python), '-mvenv', envdir, cwd='/')
+
+
+_interface = python.py_interface(ENVIRONMENT_DIR, make_venv)
+in_env, healthy, run_hook, install_environment = _interface
diff --git a/pre_commit/languages/ruby.py b/pre_commit/languages/ruby.py
new file mode 100644
index 0000000..61241f8
--- /dev/null
+++ b/pre_commit/languages/ruby.py
@@ -0,0 +1,126 @@
+import contextlib
+import os.path
+import shutil
+import tarfile
+from typing import Generator
+from typing import Sequence
+from typing import Tuple
+
+import pre_commit.constants as C
+from pre_commit.envcontext import envcontext
+from pre_commit.envcontext import PatchesT
+from pre_commit.envcontext import Var
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+from pre_commit.prefix import Prefix
+from pre_commit.util import CalledProcessError
+from pre_commit.util import clean_path_on_failure
+from pre_commit.util import resource_bytesio
+
+ENVIRONMENT_DIR = 'rbenv'
+get_default_version = helpers.basic_get_default_version
+healthy = helpers.basic_healthy
+
+
+def get_env_patch(
+ venv: str,
+ language_version: str,
+) -> PatchesT: # pragma: win32 no cover
+ patches: PatchesT = (
+ ('GEM_HOME', os.path.join(venv, 'gems')),
+ ('RBENV_ROOT', venv),
+ ('BUNDLE_IGNORE_CONFIG', '1'),
+ (
+ 'PATH', (
+ os.path.join(venv, 'gems', 'bin'), os.pathsep,
+ os.path.join(venv, 'shims'), os.pathsep,
+ os.path.join(venv, 'bin'), os.pathsep, Var('PATH'),
+ ),
+ ),
+ )
+ if language_version != C.DEFAULT:
+ patches += (('RBENV_VERSION', language_version),)
+ return patches
+
+
+@contextlib.contextmanager # pragma: win32 no cover
+def in_env(
+ prefix: Prefix,
+ language_version: str,
+) -> Generator[None, None, None]:
+ envdir = prefix.path(
+ helpers.environment_dir(ENVIRONMENT_DIR, language_version),
+ )
+ with envcontext(get_env_patch(envdir, language_version)):
+ yield
+
+
+def _extract_resource(filename: str, dest: str) -> None:
+ with resource_bytesio(filename) as bio:
+ with tarfile.open(fileobj=bio) as tf:
+ tf.extractall(dest)
+
+
+def _install_rbenv(
+ prefix: Prefix,
+ version: str = C.DEFAULT,
+) -> None: # pragma: win32 no cover
+ directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
+
+ _extract_resource('rbenv.tar.gz', prefix.path('.'))
+ shutil.move(prefix.path('rbenv'), prefix.path(directory))
+
+ # Only install ruby-build if the version is specified
+ if version != C.DEFAULT:
+ plugins_dir = prefix.path(directory, 'plugins')
+ _extract_resource('ruby-download.tar.gz', plugins_dir)
+ _extract_resource('ruby-build.tar.gz', plugins_dir)
+
+
+def _install_ruby(
+ prefix: Prefix,
+ version: str,
+) -> None: # pragma: win32 no cover
+ try:
+ helpers.run_setup_cmd(prefix, ('rbenv', 'download', version))
+ except CalledProcessError: # pragma: no cover (usually find with download)
+ # Failed to download from mirror for some reason, build it instead
+ helpers.run_setup_cmd(prefix, ('rbenv', 'install', version))
+
+
+def install_environment(
+ prefix: Prefix, version: str, additional_dependencies: Sequence[str],
+) -> None: # pragma: win32 no cover
+ additional_dependencies = tuple(additional_dependencies)
+ directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
+ with clean_path_on_failure(prefix.path(directory)):
+ # TODO: this currently will fail if there's no version specified and
+ # there's no system ruby installed. Is this ok?
+ _install_rbenv(prefix, version=version)
+ with in_env(prefix, version):
+ # Need to call this before installing so rbenv's directories are
+ # set up
+ helpers.run_setup_cmd(prefix, ('rbenv', 'init', '-'))
+ if version != C.DEFAULT:
+ _install_ruby(prefix, version)
+ # Need to call this after installing to set up the shims
+ helpers.run_setup_cmd(prefix, ('rbenv', 'rehash'))
+ helpers.run_setup_cmd(
+ prefix, ('gem', 'build', *prefix.star('.gemspec')),
+ )
+ helpers.run_setup_cmd(
+ prefix,
+ (
+ 'gem', 'install', '--no-document',
+ *prefix.star('.gem'), *additional_dependencies,
+ ),
+ )
+
+
+def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+) -> Tuple[int, bytes]: # pragma: win32 no cover
+ with in_env(hook.prefix, hook.language_version):
+ return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
diff --git a/pre_commit/languages/rust.py b/pre_commit/languages/rust.py
new file mode 100644
index 0000000..7ea3f54
--- /dev/null
+++ b/pre_commit/languages/rust.py
@@ -0,0 +1,106 @@
+import contextlib
+import os.path
+from typing import Generator
+from typing import Sequence
+from typing import Set
+from typing import Tuple
+
+import toml
+
+import pre_commit.constants as C
+from pre_commit.envcontext import envcontext
+from pre_commit.envcontext import PatchesT
+from pre_commit.envcontext import Var
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+from pre_commit.prefix import Prefix
+from pre_commit.util import clean_path_on_failure
+from pre_commit.util import cmd_output_b
+
+ENVIRONMENT_DIR = 'rustenv'
+get_default_version = helpers.basic_get_default_version
+healthy = helpers.basic_healthy
+
+
+def get_env_patch(target_dir: str) -> PatchesT:
+ return (
+ ('PATH', (os.path.join(target_dir, 'bin'), os.pathsep, Var('PATH'))),
+ )
+
+
+@contextlib.contextmanager
+def in_env(prefix: Prefix) -> Generator[None, None, None]:
+ target_dir = prefix.path(
+ helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
+ )
+ with envcontext(get_env_patch(target_dir)):
+ yield
+
+
+def _add_dependencies(
+ cargo_toml_path: str,
+ additional_dependencies: Set[str],
+) -> None:
+ with open(cargo_toml_path, 'r+') as f:
+ cargo_toml = toml.load(f)
+ cargo_toml.setdefault('dependencies', {})
+ for dep in additional_dependencies:
+ name, _, spec = dep.partition(':')
+ cargo_toml['dependencies'][name] = spec or '*'
+ f.seek(0)
+ toml.dump(cargo_toml, f)
+ f.truncate()
+
+
+def install_environment(
+ prefix: Prefix,
+ version: str,
+ additional_dependencies: Sequence[str],
+) -> None:
+ helpers.assert_version_default('rust', version)
+ directory = prefix.path(
+ helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
+ )
+
+ # There are two cases where we might want to specify more dependencies:
+ # as dependencies for the library being built, and as binary packages
+ # to be `cargo install`'d.
+ #
+ # Unlike e.g. Python, if we just `cargo install` a library, it won't be
+ # used for compilation. And if we add a crate providing a binary to the
+ # `Cargo.toml`, the binary won't be built.
+ #
+ # Because of this, we allow specifying "cli" dependencies by prefixing
+ # with 'cli:'.
+ cli_deps = {
+ dep for dep in additional_dependencies if dep.startswith('cli:')
+ }
+ lib_deps = set(additional_dependencies) - cli_deps
+
+ if len(lib_deps) > 0:
+ _add_dependencies(prefix.path('Cargo.toml'), lib_deps)
+
+ with clean_path_on_failure(directory):
+ packages_to_install: Set[Tuple[str, ...]] = {('--path', '.')}
+ for cli_dep in cli_deps:
+ cli_dep = cli_dep[len('cli:'):]
+ package, _, version = cli_dep.partition(':')
+ if version != '':
+ packages_to_install.add((package, '--version', version))
+ else:
+ packages_to_install.add((package,))
+
+ for args in packages_to_install:
+ cmd_output_b(
+ 'cargo', 'install', '--bins', '--root', directory, *args,
+ cwd=prefix.prefix_dir,
+ )
+
+
+def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+) -> Tuple[int, bytes]:
+ with in_env(hook.prefix):
+ return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
diff --git a/pre_commit/languages/script.py b/pre_commit/languages/script.py
new file mode 100644
index 0000000..a5e1365
--- /dev/null
+++ b/pre_commit/languages/script.py
@@ -0,0 +1,19 @@
+from typing import Sequence
+from typing import Tuple
+
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+
+ENVIRONMENT_DIR = None
+get_default_version = helpers.basic_get_default_version
+healthy = helpers.basic_healthy
+install_environment = helpers.no_install
+
+
+def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+) -> Tuple[int, bytes]:
+ cmd = (hook.prefix.path(hook.cmd[0]), *hook.cmd[1:])
+ return helpers.run_xargs(hook, cmd, file_args, color=color)
diff --git a/pre_commit/languages/swift.py b/pre_commit/languages/swift.py
new file mode 100644
index 0000000..66aadc8
--- /dev/null
+++ b/pre_commit/languages/swift.py
@@ -0,0 +1,64 @@
+import contextlib
+import os
+from typing import Generator
+from typing import Sequence
+from typing import Tuple
+
+import pre_commit.constants as C
+from pre_commit.envcontext import envcontext
+from pre_commit.envcontext import PatchesT
+from pre_commit.envcontext import Var
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+from pre_commit.prefix import Prefix
+from pre_commit.util import clean_path_on_failure
+from pre_commit.util import cmd_output_b
+
+ENVIRONMENT_DIR = 'swift_env'
+get_default_version = helpers.basic_get_default_version
+healthy = helpers.basic_healthy
+BUILD_DIR = '.build'
+BUILD_CONFIG = 'release'
+
+
+def get_env_patch(venv: str) -> PatchesT: # pragma: win32 no cover
+ bin_path = os.path.join(venv, BUILD_DIR, BUILD_CONFIG)
+ return (('PATH', (bin_path, os.pathsep, Var('PATH'))),)
+
+
+@contextlib.contextmanager # pragma: win32 no cover
+def in_env(prefix: Prefix) -> Generator[None, None, None]:
+ envdir = prefix.path(
+ helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
+ )
+ with envcontext(get_env_patch(envdir)):
+ yield
+
+
+def install_environment(
+ prefix: Prefix, version: str, additional_dependencies: Sequence[str],
+) -> None: # pragma: win32 no cover
+ helpers.assert_version_default('swift', version)
+ helpers.assert_no_additional_deps('swift', additional_dependencies)
+ directory = prefix.path(
+ helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
+ )
+
+ # Build the swift package
+ with clean_path_on_failure(directory):
+ os.mkdir(directory)
+ cmd_output_b(
+ 'swift', 'build',
+ '-C', prefix.prefix_dir,
+ '-c', BUILD_CONFIG,
+ '--build-path', os.path.join(directory, BUILD_DIR),
+ )
+
+
+def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+) -> Tuple[int, bytes]: # pragma: win32 no cover
+ with in_env(hook.prefix):
+ return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
diff --git a/pre_commit/languages/system.py b/pre_commit/languages/system.py
new file mode 100644
index 0000000..139f45d
--- /dev/null
+++ b/pre_commit/languages/system.py
@@ -0,0 +1,19 @@
+from typing import Sequence
+from typing import Tuple
+
+from pre_commit.hook import Hook
+from pre_commit.languages import helpers
+
+
+ENVIRONMENT_DIR = None
+get_default_version = helpers.basic_get_default_version
+healthy = helpers.basic_healthy
+install_environment = helpers.no_install
+
+
+def run_hook(
+ hook: Hook,
+ file_args: Sequence[str],
+ color: bool,
+) -> Tuple[int, bytes]:
+ return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
diff --git a/pre_commit/logging_handler.py b/pre_commit/logging_handler.py
new file mode 100644
index 0000000..ba05295
--- /dev/null
+++ b/pre_commit/logging_handler.py
@@ -0,0 +1,40 @@
+import contextlib
+import logging
+from typing import Generator
+
+from pre_commit import color
+from pre_commit import output
+
+logger = logging.getLogger('pre_commit')
+
+LOG_LEVEL_COLORS = {
+ 'DEBUG': '',
+ 'INFO': '',
+ 'WARNING': color.YELLOW,
+ 'ERROR': color.RED,
+}
+
+
+class LoggingHandler(logging.Handler):
+ def __init__(self, use_color: bool) -> None:
+ super().__init__()
+ self.use_color = use_color
+
+ def emit(self, record: logging.LogRecord) -> None:
+ level_msg = color.format_color(
+ f'[{record.levelname}]',
+ LOG_LEVEL_COLORS[record.levelname],
+ self.use_color,
+ )
+ output.write_line(f'{level_msg} {record.getMessage()}')
+
+
+@contextlib.contextmanager
+def logging_handler(use_color: bool) -> Generator[None, None, None]:
+ handler = LoggingHandler(use_color)
+ logger.addHandler(handler)
+ logger.setLevel(logging.INFO)
+ try:
+ yield
+ finally:
+ logger.removeHandler(handler)
diff --git a/pre_commit/main.py b/pre_commit/main.py
new file mode 100644
index 0000000..790b347
--- /dev/null
+++ b/pre_commit/main.py
@@ -0,0 +1,410 @@
+import argparse
+import logging
+import os
+import sys
+from typing import Any
+from typing import Optional
+from typing import Sequence
+from typing import Union
+
+import pre_commit.constants as C
+from pre_commit import color
+from pre_commit import git
+from pre_commit.commands.autoupdate import autoupdate
+from pre_commit.commands.clean import clean
+from pre_commit.commands.gc import gc
+from pre_commit.commands.hook_impl import hook_impl
+from pre_commit.commands.init_templatedir import init_templatedir
+from pre_commit.commands.install_uninstall import install
+from pre_commit.commands.install_uninstall import install_hooks
+from pre_commit.commands.install_uninstall import uninstall
+from pre_commit.commands.migrate_config import migrate_config
+from pre_commit.commands.run import run
+from pre_commit.commands.sample_config import sample_config
+from pre_commit.commands.try_repo import try_repo
+from pre_commit.error_handler import error_handler
+from pre_commit.error_handler import FatalError
+from pre_commit.logging_handler import logging_handler
+from pre_commit.store import Store
+from pre_commit.util import CalledProcessError
+
+
+logger = logging.getLogger('pre_commit')
+
+# https://github.com/pre-commit/pre-commit/issues/217
+# On OSX, making a virtualenv using pyvenv at . causes `virtualenv` and `pip`
+# to install packages to the wrong place. We don't want anything to deal with
+# pyvenv
+os.environ.pop('__PYVENV_LAUNCHER__', None)
+
+
+COMMANDS_NO_GIT = {'clean', 'gc', 'init-templatedir', 'sample-config'}
+
+
+def _add_color_option(parser: argparse.ArgumentParser) -> None:
+ parser.add_argument(
+ '--color', default=os.environ.get('PRE_COMMIT_COLOR', 'auto'),
+ type=color.use_color,
+ metavar='{' + ','.join(color.COLOR_CHOICES) + '}',
+ help='Whether to use color in output. Defaults to `%(default)s`.',
+ )
+
+
+def _add_config_option(parser: argparse.ArgumentParser) -> None:
+ parser.add_argument(
+ '-c', '--config', default=C.CONFIG_FILE,
+ help='Path to alternate config file',
+ )
+
+
+class AppendReplaceDefault(argparse.Action):
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ super().__init__(*args, **kwargs)
+ self.appended = False
+
+ def __call__(
+ self,
+ parser: argparse.ArgumentParser,
+ namespace: argparse.Namespace,
+ values: Union[str, Sequence[str], None],
+ option_string: Optional[str] = None,
+ ) -> None:
+ if not self.appended:
+ setattr(namespace, self.dest, [])
+ self.appended = True
+ getattr(namespace, self.dest).append(values)
+
+
+def _add_hook_type_option(parser: argparse.ArgumentParser) -> None:
+ parser.add_argument(
+ '-t', '--hook-type', choices=(
+ 'pre-commit', 'pre-merge-commit', 'pre-push',
+ 'prepare-commit-msg', 'commit-msg', 'post-checkout',
+ ),
+ action=AppendReplaceDefault,
+ default=['pre-commit'],
+ dest='hook_types',
+ )
+
+
+def _add_run_options(parser: argparse.ArgumentParser) -> None:
+ parser.add_argument('hook', nargs='?', help='A single hook-id to run')
+ parser.add_argument('--verbose', '-v', action='store_true', default=False)
+ mutex_group = parser.add_mutually_exclusive_group(required=False)
+ mutex_group.add_argument(
+ '--all-files', '-a', action='store_true', default=False,
+ help='Run on all the files in the repo.',
+ )
+ mutex_group.add_argument(
+ '--files', nargs='*', default=[],
+ help='Specific filenames to run hooks on.',
+ )
+ parser.add_argument(
+ '--show-diff-on-failure', action='store_true',
+ help='When hooks fail, run `git diff` directly afterward.',
+ )
+ parser.add_argument(
+ '--hook-stage', choices=C.STAGES, default='commit',
+ help='The stage during which the hook is fired. One of %(choices)s',
+ )
+ parser.add_argument(
+ '--from-ref', '--source', '-s',
+ help=(
+ '(for usage with `--from-ref`) -- this option represents the '
+ 'original ref in a `from_ref...to_ref` diff expression. '
+ 'For `pre-push` hooks, this represents the branch you are pushing '
+ 'to. '
+ 'For `post-checkout` hooks, this represents the branch that was '
+ 'previously checked out.'
+ ),
+ )
+ parser.add_argument(
+ '--to-ref', '--origin', '-o',
+ help=(
+ '(for usage with `--to-ref`) -- this option represents the '
+ 'destination ref in a `from_ref...to_ref` diff expression. '
+ 'For `pre-push` hooks, this represents the branch being pushed. '
+ 'For `post-checkout` hooks, this represents the branch that is '
+ 'now checked out.'
+ ),
+ )
+ parser.add_argument(
+ '--commit-msg-filename',
+ help='Filename to check when running during `commit-msg`',
+ )
+ parser.add_argument(
+ '--remote-name', help='Remote name used by `git push`.',
+ )
+ parser.add_argument('--remote-url', help='Remote url used by `git push`.')
+ parser.add_argument(
+ '--checkout-type',
+ help=(
+ 'Indicates whether the checkout was a branch checkout '
+ '(changing branches, flag=1) or a file checkout (retrieving a '
+ 'file from the index, flag=0).'
+ ),
+ )
+
+
+def _adjust_args_and_chdir(args: argparse.Namespace) -> None:
+ # `--config` was specified relative to the non-root working directory
+ if os.path.exists(args.config):
+ args.config = os.path.abspath(args.config)
+ if args.command in {'run', 'try-repo'}:
+ args.files = [os.path.abspath(filename) for filename in args.files]
+ if args.command == 'try-repo' and os.path.exists(args.repo):
+ args.repo = os.path.abspath(args.repo)
+
+ try:
+ toplevel = git.get_root()
+ except CalledProcessError:
+ raise FatalError(
+ 'git failed. Is it installed, and are you in a Git repository '
+ 'directory?',
+ )
+ else:
+ if toplevel == '': # pragma: no cover (old git)
+ raise FatalError(
+ 'git toplevel unexpectedly empty! make sure you are not '
+ 'inside the `.git` directory of your repository.',
+ )
+ else:
+ os.chdir(toplevel)
+
+ args.config = os.path.relpath(args.config)
+ if args.command in {'run', 'try-repo'}:
+ args.files = [os.path.relpath(filename) for filename in args.files]
+ if args.command == 'try-repo' and os.path.exists(args.repo):
+ args.repo = os.path.relpath(args.repo)
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+ argv = argv if argv is not None else sys.argv[1:]
+ parser = argparse.ArgumentParser(prog='pre-commit')
+
+ # https://stackoverflow.com/a/8521644/812183
+ parser.add_argument(
+ '-V', '--version',
+ action='version',
+ version=f'%(prog)s {C.VERSION}',
+ )
+
+ subparsers = parser.add_subparsers(dest='command')
+
+ autoupdate_parser = subparsers.add_parser(
+ 'autoupdate',
+ help="Auto-update pre-commit config to the latest repos' versions.",
+ )
+ _add_color_option(autoupdate_parser)
+ _add_config_option(autoupdate_parser)
+ autoupdate_parser.add_argument(
+ '--bleeding-edge', action='store_true',
+ help=(
+ 'Update to the bleeding edge of `master` instead of the latest '
+ 'tagged version (the default behavior).'
+ ),
+ )
+ autoupdate_parser.add_argument(
+ '--freeze', action='store_true',
+ help='Store "frozen" hashes in `rev` instead of tag names',
+ )
+ autoupdate_parser.add_argument(
+ '--repo', dest='repos', action='append', metavar='REPO',
+ help='Only update this repository -- may be specified multiple times.',
+ )
+
+ clean_parser = subparsers.add_parser(
+ 'clean', help='Clean out pre-commit files.',
+ )
+ _add_color_option(clean_parser)
+ _add_config_option(clean_parser)
+
+ hook_impl_parser = subparsers.add_parser('hook-impl')
+ _add_color_option(hook_impl_parser)
+ _add_config_option(hook_impl_parser)
+ hook_impl_parser.add_argument('--hook-type')
+ hook_impl_parser.add_argument('--hook-dir')
+ hook_impl_parser.add_argument(
+ '--skip-on-missing-config', action='store_true',
+ )
+ hook_impl_parser.add_argument(dest='rest', nargs=argparse.REMAINDER)
+
+ gc_parser = subparsers.add_parser('gc', help='Clean unused cached repos.')
+ _add_color_option(gc_parser)
+ _add_config_option(gc_parser)
+
+ init_templatedir_parser = subparsers.add_parser(
+ 'init-templatedir',
+ help=(
+ 'Install hook script in a directory intended for use with '
+ '`git config init.templateDir`.'
+ ),
+ )
+ _add_color_option(init_templatedir_parser)
+ _add_config_option(init_templatedir_parser)
+ init_templatedir_parser.add_argument(
+ 'directory', help='The directory in which to write the hook script.',
+ )
+ _add_hook_type_option(init_templatedir_parser)
+
+ install_parser = subparsers.add_parser(
+ 'install', help='Install the pre-commit script.',
+ )
+ _add_color_option(install_parser)
+ _add_config_option(install_parser)
+ install_parser.add_argument(
+ '-f', '--overwrite', action='store_true',
+ help='Overwrite existing hooks / remove migration mode.',
+ )
+ install_parser.add_argument(
+ '--install-hooks', action='store_true',
+ help=(
+ 'Whether to install hook environments for all environments '
+ 'in the config file.'
+ ),
+ )
+ _add_hook_type_option(install_parser)
+ install_parser.add_argument(
+ '--allow-missing-config', action='store_true', default=False,
+ help=(
+ 'Whether to allow a missing `pre-commit` configuration file '
+ 'or exit with a failure code.'
+ ),
+ )
+
+ install_hooks_parser = subparsers.add_parser(
+ 'install-hooks',
+ help=(
+ 'Install hook environments for all environments in the config '
+ 'file. You may find `pre-commit install --install-hooks` more '
+ 'useful.'
+ ),
+ )
+ _add_color_option(install_hooks_parser)
+ _add_config_option(install_hooks_parser)
+
+ migrate_config_parser = subparsers.add_parser(
+ 'migrate-config',
+ help='Migrate list configuration to new map configuration.',
+ )
+ _add_color_option(migrate_config_parser)
+ _add_config_option(migrate_config_parser)
+
+ run_parser = subparsers.add_parser('run', help='Run hooks.')
+ _add_color_option(run_parser)
+ _add_config_option(run_parser)
+ _add_run_options(run_parser)
+
+ sample_config_parser = subparsers.add_parser(
+ 'sample-config', help=f'Produce a sample {C.CONFIG_FILE} file',
+ )
+ _add_color_option(sample_config_parser)
+ _add_config_option(sample_config_parser)
+
+ try_repo_parser = subparsers.add_parser(
+ 'try-repo',
+ help='Try the hooks in a repository, useful for developing new hooks.',
+ )
+ _add_color_option(try_repo_parser)
+ _add_config_option(try_repo_parser)
+ try_repo_parser.add_argument(
+ 'repo', help='Repository to source hooks from.',
+ )
+ try_repo_parser.add_argument(
+ '--ref', '--rev',
+ help=(
+ 'Manually select a rev to run against, otherwise the `HEAD` '
+ 'revision will be used.'
+ ),
+ )
+ _add_run_options(try_repo_parser)
+
+ uninstall_parser = subparsers.add_parser(
+ 'uninstall', help='Uninstall the pre-commit script.',
+ )
+ _add_color_option(uninstall_parser)
+ _add_config_option(uninstall_parser)
+ _add_hook_type_option(uninstall_parser)
+
+ help = subparsers.add_parser(
+ 'help', help='Show help for a specific command.',
+ )
+ help.add_argument('help_cmd', nargs='?', help='Command to show help for.')
+
+ # argparse doesn't really provide a way to use a `default` subparser
+ if len(argv) == 0:
+ argv = ['run']
+ args = parser.parse_args(argv)
+
+ if args.command == 'help' and args.help_cmd:
+ parser.parse_args([args.help_cmd, '--help'])
+ elif args.command == 'help':
+ parser.parse_args(['--help'])
+
+ with error_handler(), logging_handler(args.color):
+ if args.command not in COMMANDS_NO_GIT:
+ _adjust_args_and_chdir(args)
+
+ git.check_for_cygwin_mismatch()
+
+ store = Store()
+ store.mark_config_used(args.config)
+
+ if args.command == 'autoupdate':
+ return autoupdate(
+ args.config, store,
+ tags_only=not args.bleeding_edge,
+ freeze=args.freeze,
+ repos=args.repos,
+ )
+ elif args.command == 'clean':
+ return clean(store)
+ elif args.command == 'gc':
+ return gc(store)
+ elif args.command == 'hook-impl':
+ return hook_impl(
+ store,
+ config=args.config,
+ color=args.color,
+ hook_type=args.hook_type,
+ hook_dir=args.hook_dir,
+ skip_on_missing_config=args.skip_on_missing_config,
+ args=args.rest[1:],
+ )
+ elif args.command == 'install':
+ return install(
+ args.config, store,
+ hook_types=args.hook_types,
+ overwrite=args.overwrite,
+ hooks=args.install_hooks,
+ skip_on_missing_config=args.allow_missing_config,
+ )
+ elif args.command == 'init-templatedir':
+ return init_templatedir(
+ args.config, store, args.directory,
+ hook_types=args.hook_types,
+ )
+ elif args.command == 'install-hooks':
+ return install_hooks(args.config, store)
+ elif args.command == 'migrate-config':
+ return migrate_config(args.config)
+ elif args.command == 'run':
+ return run(args.config, store, args)
+ elif args.command == 'sample-config':
+ return sample_config()
+ elif args.command == 'try-repo':
+ return try_repo(args)
+ elif args.command == 'uninstall':
+ return uninstall(hook_types=args.hook_types)
+ else:
+ raise NotImplementedError(
+ f'Command {args.command} not implemented.',
+ )
+
+ raise AssertionError(
+ f'Command {args.command} failed to exit with a returncode',
+ )
+
+
+if __name__ == '__main__':
+ exit(main())
diff --git a/pre_commit/make_archives.py b/pre_commit/make_archives.py
new file mode 100644
index 0000000..c31bcd7
--- /dev/null
+++ b/pre_commit/make_archives.py
@@ -0,0 +1,65 @@
+import argparse
+import os.path
+import tarfile
+from typing import Optional
+from typing import Sequence
+
+from pre_commit import output
+from pre_commit.util import cmd_output_b
+from pre_commit.util import rmtree
+from pre_commit.util import tmpdir
+
+
+# This is a script for generating the tarred resources for git repo
+# dependencies. Currently it's just for "vendoring" ruby support packages.
+
+
+REPOS = (
+ ('rbenv', 'git://github.com/rbenv/rbenv', 'a3fa9b7'),
+ ('ruby-build', 'git://github.com/rbenv/ruby-build', '1a902f3'),
+ (
+ 'ruby-download',
+ 'git://github.com/garnieretienne/rvm-download',
+ '09bd7c6',
+ ),
+)
+
+
+def make_archive(name: str, repo: str, ref: str, destdir: str) -> str:
+ """Makes an archive of a repository in the given destdir.
+
+ :param text name: Name to give the archive. For instance foo. The file
+ that is created will be called foo.tar.gz.
+ :param text repo: Repository to clone.
+ :param text ref: Tag/SHA/branch to check out.
+ :param text destdir: Directory to place archives in.
+ """
+ output_path = os.path.join(destdir, f'{name}.tar.gz')
+ with tmpdir() as tempdir:
+ # Clone the repository to the temporary directory
+ cmd_output_b('git', 'clone', repo, tempdir)
+ cmd_output_b('git', 'checkout', ref, cwd=tempdir)
+
+ # We don't want the '.git' directory
+ # It adds a bunch of size to the archive and we don't use it at
+ # runtime
+ rmtree(os.path.join(tempdir, '.git'))
+
+ with tarfile.open(output_path, 'w|gz') as tf:
+ tf.add(tempdir, name)
+
+ return output_path
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--dest', default='pre_commit/resources')
+ args = parser.parse_args(argv)
+ for archive_name, repo, ref in REPOS:
+ output.write_line(f'Making {archive_name}.tar.gz for {repo}@{ref}')
+ make_archive(archive_name, repo, ref, args.dest)
+ return 0
+
+
+if __name__ == '__main__':
+ exit(main())
diff --git a/pre_commit/meta_hooks/__init__.py b/pre_commit/meta_hooks/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/pre_commit/meta_hooks/__init__.py
diff --git a/pre_commit/meta_hooks/check_hooks_apply.py b/pre_commit/meta_hooks/check_hooks_apply.py
new file mode 100644
index 0000000..d0244a9
--- /dev/null
+++ b/pre_commit/meta_hooks/check_hooks_apply.py
@@ -0,0 +1,39 @@
+import argparse
+from typing import Optional
+from typing import Sequence
+
+import pre_commit.constants as C
+from pre_commit import git
+from pre_commit.clientlib import load_config
+from pre_commit.commands.run import Classifier
+from pre_commit.repository import all_hooks
+from pre_commit.store import Store
+
+
+def check_all_hooks_match_files(config_file: str) -> int:
+ classifier = Classifier(git.get_all_files())
+ retv = 0
+
+ for hook in all_hooks(load_config(config_file), Store()):
+ if hook.always_run or hook.language == 'fail':
+ continue
+ elif not classifier.filenames_for_hook(hook):
+ print(f'{hook.id} does not apply to this repository')
+ retv = 1
+
+ return retv
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument('filenames', nargs='*', default=[C.CONFIG_FILE])
+ args = parser.parse_args(argv)
+
+ retv = 0
+ for filename in args.filenames:
+ retv |= check_all_hooks_match_files(filename)
+ return retv
+
+
+if __name__ == '__main__':
+ exit(main())
diff --git a/pre_commit/meta_hooks/check_useless_excludes.py b/pre_commit/meta_hooks/check_useless_excludes.py
new file mode 100644
index 0000000..30b8d81
--- /dev/null
+++ b/pre_commit/meta_hooks/check_useless_excludes.py
@@ -0,0 +1,72 @@
+import argparse
+import re
+from typing import Optional
+from typing import Sequence
+
+from cfgv import apply_defaults
+
+import pre_commit.constants as C
+from pre_commit import git
+from pre_commit.clientlib import load_config
+from pre_commit.clientlib import MANIFEST_HOOK_DICT
+from pre_commit.commands.run import Classifier
+
+
+def exclude_matches_any(
+ filenames: Sequence[str],
+ include: str,
+ exclude: str,
+) -> bool:
+ if exclude == '^$':
+ return True
+ include_re, exclude_re = re.compile(include), re.compile(exclude)
+ for filename in filenames:
+ if include_re.search(filename) and exclude_re.search(filename):
+ return True
+ return False
+
+
+def check_useless_excludes(config_file: str) -> int:
+ config = load_config(config_file)
+ classifier = Classifier(git.get_all_files())
+ retv = 0
+
+ exclude = config['exclude']
+ if not exclude_matches_any(classifier.filenames, '', exclude):
+ print(
+ f'The global exclude pattern {exclude!r} does not match any files',
+ )
+ retv = 1
+
+ for repo in config['repos']:
+ for hook in repo['hooks']:
+ # Not actually a manifest dict, but this more accurately reflects
+ # the defaults applied during runtime
+ hook = apply_defaults(hook, MANIFEST_HOOK_DICT)
+ names = classifier.filenames
+ types, exclude_types = hook['types'], hook['exclude_types']
+ names = classifier.by_types(names, types, exclude_types)
+ include, exclude = hook['files'], hook['exclude']
+ if not exclude_matches_any(names, include, exclude):
+ print(
+ f'The exclude pattern {exclude!r} for {hook["id"]} does '
+ f'not match any files',
+ )
+ retv = 1
+
+ return retv
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument('filenames', nargs='*', default=[C.CONFIG_FILE])
+ args = parser.parse_args(argv)
+
+ retv = 0
+ for filename in args.filenames:
+ retv |= check_useless_excludes(filename)
+ return retv
+
+
+if __name__ == '__main__':
+ exit(main())
diff --git a/pre_commit/meta_hooks/identity.py b/pre_commit/meta_hooks/identity.py
new file mode 100644
index 0000000..730d0ec
--- /dev/null
+++ b/pre_commit/meta_hooks/identity.py
@@ -0,0 +1,16 @@
+import sys
+from typing import Optional
+from typing import Sequence
+
+from pre_commit import output
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+ argv = argv if argv is not None else sys.argv[1:]
+ for arg in argv:
+ output.write_line(arg)
+ return 0
+
+
+if __name__ == '__main__':
+ exit(main())
diff --git a/pre_commit/output.py b/pre_commit/output.py
new file mode 100644
index 0000000..24f9d84
--- /dev/null
+++ b/pre_commit/output.py
@@ -0,0 +1,32 @@
+import contextlib
+import sys
+from typing import Any
+from typing import IO
+from typing import Optional
+
+
+def write(s: str, stream: IO[bytes] = sys.stdout.buffer) -> None:
+ stream.write(s.encode())
+ stream.flush()
+
+
+def write_line_b(
+ s: Optional[bytes] = None,
+ stream: IO[bytes] = sys.stdout.buffer,
+ logfile_name: Optional[str] = None,
+) -> None:
+ with contextlib.ExitStack() as exit_stack:
+ output_streams = [stream]
+ if logfile_name:
+ stream = exit_stack.enter_context(open(logfile_name, 'ab'))
+ output_streams.append(stream)
+
+ for output_stream in output_streams:
+ if s is not None:
+ output_stream.write(s)
+ output_stream.write(b'\n')
+ output_stream.flush()
+
+
+def write_line(s: Optional[str] = None, **kwargs: Any) -> None:
+ write_line_b(s.encode() if s is not None else s, **kwargs)
diff --git a/pre_commit/parse_shebang.py b/pre_commit/parse_shebang.py
new file mode 100644
index 0000000..d344a1d
--- /dev/null
+++ b/pre_commit/parse_shebang.py
@@ -0,0 +1,84 @@
+import os.path
+from typing import Mapping
+from typing import Optional
+from typing import Tuple
+from typing import TYPE_CHECKING
+
+from identify.identify import parse_shebang_from_file
+
+if TYPE_CHECKING:
+ from typing import NoReturn
+
+
+class ExecutableNotFoundError(OSError):
+ def to_output(self) -> Tuple[int, bytes, None]:
+ return (1, self.args[0].encode(), None)
+
+
+def parse_filename(filename: str) -> Tuple[str, ...]:
+ if not os.path.exists(filename):
+ return ()
+ else:
+ return parse_shebang_from_file(filename)
+
+
+def find_executable(
+ exe: str, _environ: Optional[Mapping[str, str]] = None,
+) -> Optional[str]:
+ exe = os.path.normpath(exe)
+ if os.sep in exe:
+ return exe
+
+ environ = _environ if _environ is not None else os.environ
+
+ if 'PATHEXT' in environ:
+ exts = environ['PATHEXT'].split(os.pathsep)
+ possible_exe_names = tuple(f'{exe}{ext}' for ext in exts) + (exe,)
+ else:
+ possible_exe_names = (exe,)
+
+ for path in environ.get('PATH', '').split(os.pathsep):
+ for possible_exe_name in possible_exe_names:
+ joined = os.path.join(path, possible_exe_name)
+ if os.path.isfile(joined) and os.access(joined, os.X_OK):
+ return joined
+ else:
+ return None
+
+
+def normexe(orig: str) -> str:
+ def _error(msg: str) -> 'NoReturn':
+ raise ExecutableNotFoundError(f'Executable `{orig}` {msg}')
+
+ if os.sep not in orig and (not os.altsep or os.altsep not in orig):
+ exe = find_executable(orig)
+ if exe is None:
+ _error('not found')
+ return exe
+ elif os.path.isdir(orig):
+ _error('is a directory')
+ elif not os.path.isfile(orig):
+ _error('not found')
+ elif not os.access(orig, os.X_OK): # pragma: win32 no cover
+ _error('is not executable')
+ else:
+ return orig
+
+
+def normalize_cmd(cmd: Tuple[str, ...]) -> Tuple[str, ...]:
+ """Fixes for the following issues on windows
+ - https://bugs.python.org/issue8557
+ - windows does not parse shebangs
+
+ This function also makes deep-path shebangs work just fine
+ """
+ # Use PATH to determine the executable
+ exe = normexe(cmd[0])
+
+ # Figure out the shebang from the resulting command
+ cmd = parse_filename(exe) + (exe,) + cmd[1:]
+
+ # This could have given us back another bare executable
+ exe = normexe(cmd[0])
+
+ return (exe,) + cmd[1:]
diff --git a/pre_commit/prefix.py b/pre_commit/prefix.py
new file mode 100644
index 0000000..0e3ebbd
--- /dev/null
+++ b/pre_commit/prefix.py
@@ -0,0 +1,17 @@
+import os.path
+from typing import NamedTuple
+from typing import Tuple
+
+
+class Prefix(NamedTuple):
+ prefix_dir: str
+
+ def path(self, *parts: str) -> str:
+ return os.path.normpath(os.path.join(self.prefix_dir, *parts))
+
+ def exists(self, *parts: str) -> bool:
+ return os.path.exists(self.path(*parts))
+
+ def star(self, end: str) -> Tuple[str, ...]:
+ paths = os.listdir(self.prefix_dir)
+ return tuple(path for path in paths if path.endswith(end))
diff --git a/pre_commit/repository.py b/pre_commit/repository.py
new file mode 100644
index 0000000..77734ee
--- /dev/null
+++ b/pre_commit/repository.py
@@ -0,0 +1,208 @@
+import json
+import logging
+import os
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Sequence
+from typing import Set
+from typing import Tuple
+
+import pre_commit.constants as C
+from pre_commit.clientlib import load_manifest
+from pre_commit.clientlib import LOCAL
+from pre_commit.clientlib import META
+from pre_commit.hook import Hook
+from pre_commit.languages.all import languages
+from pre_commit.languages.helpers import environment_dir
+from pre_commit.prefix import Prefix
+from pre_commit.store import Store
+from pre_commit.util import parse_version
+from pre_commit.util import rmtree
+
+
+logger = logging.getLogger('pre_commit')
+
+
+def _state(additional_deps: Sequence[str]) -> object:
+ return {'additional_dependencies': sorted(additional_deps)}
+
+
+def _state_filename(prefix: Prefix, venv: str) -> str:
+ return prefix.path(venv, f'.install_state_v{C.INSTALLED_STATE_VERSION}')
+
+
+def _read_state(prefix: Prefix, venv: str) -> Optional[object]:
+ filename = _state_filename(prefix, venv)
+ if not os.path.exists(filename):
+ return None
+ else:
+ with open(filename) as f:
+ return json.load(f)
+
+
+def _write_state(prefix: Prefix, venv: str, state: object) -> None:
+ state_filename = _state_filename(prefix, venv)
+ staging = f'{state_filename}staging'
+ with open(staging, 'w') as state_file:
+ state_file.write(json.dumps(state))
+ # Move the file into place atomically to indicate we've installed
+ os.rename(staging, state_filename)
+
+
+def _hook_installed(hook: Hook) -> bool:
+ lang = languages[hook.language]
+ venv = environment_dir(lang.ENVIRONMENT_DIR, hook.language_version)
+ return (
+ venv is None or (
+ (
+ _read_state(hook.prefix, venv) ==
+ _state(hook.additional_dependencies)
+ ) and
+ lang.healthy(hook.prefix, hook.language_version)
+ )
+ )
+
+
+def _hook_install(hook: Hook) -> None:
+ logger.info(f'Installing environment for {hook.src}.')
+ logger.info('Once installed this environment will be reused.')
+ logger.info('This may take a few minutes...')
+
+ lang = languages[hook.language]
+ assert lang.ENVIRONMENT_DIR is not None
+ venv = environment_dir(lang.ENVIRONMENT_DIR, hook.language_version)
+
+ # There's potentially incomplete cleanup from previous runs
+ # Clean it up!
+ if hook.prefix.exists(venv):
+ rmtree(hook.prefix.path(venv))
+
+ lang.install_environment(
+ hook.prefix, hook.language_version, hook.additional_dependencies,
+ )
+ # Write our state to indicate we're installed
+ _write_state(hook.prefix, venv, _state(hook.additional_dependencies))
+
+
+def _hook(
+ *hook_dicts: Dict[str, Any],
+ root_config: Dict[str, Any],
+) -> Dict[str, Any]:
+ ret, rest = dict(hook_dicts[0]), hook_dicts[1:]
+ for dct in rest:
+ ret.update(dct)
+
+ version = ret['minimum_pre_commit_version']
+ if parse_version(version) > parse_version(C.VERSION):
+ logger.error(
+ f'The hook `{ret["id"]}` requires pre-commit version {version} '
+ f'but version {C.VERSION} is installed. '
+ f'Perhaps run `pip install --upgrade pre-commit`.',
+ )
+ exit(1)
+
+ lang = ret['language']
+ if ret['language_version'] == C.DEFAULT:
+ ret['language_version'] = root_config['default_language_version'][lang]
+ if ret['language_version'] == C.DEFAULT:
+ ret['language_version'] = languages[lang].get_default_version()
+
+ if not ret['stages']:
+ ret['stages'] = root_config['default_stages']
+
+ return ret
+
+
+def _non_cloned_repository_hooks(
+ repo_config: Dict[str, Any],
+ store: Store,
+ root_config: Dict[str, Any],
+) -> Tuple[Hook, ...]:
+ def _prefix(language_name: str, deps: Sequence[str]) -> Prefix:
+ language = languages[language_name]
+ # pygrep / script / system / docker_image do not have
+ # environments so they work out of the current directory
+ if language.ENVIRONMENT_DIR is None:
+ return Prefix(os.getcwd())
+ else:
+ return Prefix(store.make_local(deps))
+
+ return tuple(
+ Hook.create(
+ repo_config['repo'],
+ _prefix(hook['language'], hook['additional_dependencies']),
+ _hook(hook, root_config=root_config),
+ )
+ for hook in repo_config['hooks']
+ )
+
+
+def _cloned_repository_hooks(
+ repo_config: Dict[str, Any],
+ store: Store,
+ root_config: Dict[str, Any],
+) -> Tuple[Hook, ...]:
+ repo, rev = repo_config['repo'], repo_config['rev']
+ manifest_path = os.path.join(store.clone(repo, rev), C.MANIFEST_FILE)
+ by_id = {hook['id']: hook for hook in load_manifest(manifest_path)}
+
+ for hook in repo_config['hooks']:
+ if hook['id'] not in by_id:
+ logger.error(
+ f'`{hook["id"]}` is not present in repository {repo}. '
+ f'Typo? Perhaps it is introduced in a newer version? '
+ f'Often `pre-commit autoupdate` fixes this.',
+ )
+ exit(1)
+
+ hook_dcts = [
+ _hook(by_id[hook['id']], hook, root_config=root_config)
+ for hook in repo_config['hooks']
+ ]
+ return tuple(
+ Hook.create(
+ repo_config['repo'],
+ Prefix(store.clone(repo, rev, hook['additional_dependencies'])),
+ hook,
+ )
+ for hook in hook_dcts
+ )
+
+
+def _repository_hooks(
+ repo_config: Dict[str, Any],
+ store: Store,
+ root_config: Dict[str, Any],
+) -> Tuple[Hook, ...]:
+ if repo_config['repo'] in {LOCAL, META}:
+ return _non_cloned_repository_hooks(repo_config, store, root_config)
+ else:
+ return _cloned_repository_hooks(repo_config, store, root_config)
+
+
+def install_hook_envs(hooks: Sequence[Hook], store: Store) -> None:
+ def _need_installed() -> List[Hook]:
+ seen: Set[Tuple[Prefix, str, str, Tuple[str, ...]]] = set()
+ ret = []
+ for hook in hooks:
+ if hook.install_key not in seen and not _hook_installed(hook):
+ ret.append(hook)
+ seen.add(hook.install_key)
+ return ret
+
+ if not _need_installed():
+ return
+ with store.exclusive_lock():
+ # Another process may have already completed this work
+ for hook in _need_installed():
+ _hook_install(hook)
+
+
+def all_hooks(root_config: Dict[str, Any], store: Store) -> Tuple[Hook, ...]:
+ return tuple(
+ hook
+ for repo in root_config['repos']
+ for hook in _repository_hooks(repo, store, root_config)
+ )
diff --git a/pre_commit/resources/__init__.py b/pre_commit/resources/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/pre_commit/resources/__init__.py
diff --git a/pre_commit/resources/empty_template_.npmignore b/pre_commit/resources/empty_template_.npmignore
new file mode 100644
index 0000000..72e8ffc
--- /dev/null
+++ b/pre_commit/resources/empty_template_.npmignore
@@ -0,0 +1 @@
+*
diff --git a/pre_commit/resources/empty_template_Cargo.toml b/pre_commit/resources/empty_template_Cargo.toml
new file mode 100644
index 0000000..3dfeffa
--- /dev/null
+++ b/pre_commit/resources/empty_template_Cargo.toml
@@ -0,0 +1,7 @@
+[package]
+name = "__fake_crate"
+version = "0.0.0"
+
+[[bin]]
+name = "__fake_cmd"
+path = "main.rs"
diff --git a/pre_commit/resources/empty_template_Makefile.PL b/pre_commit/resources/empty_template_Makefile.PL
new file mode 100644
index 0000000..ac75fe5
--- /dev/null
+++ b/pre_commit/resources/empty_template_Makefile.PL
@@ -0,0 +1,6 @@
+use ExtUtils::MakeMaker;
+
+WriteMakefile(
+ NAME => "PreCommitDummy",
+ VERSION => "0.0.1",
+);
diff --git a/pre_commit/resources/empty_template_environment.yml b/pre_commit/resources/empty_template_environment.yml
new file mode 100644
index 0000000..0f29f0c
--- /dev/null
+++ b/pre_commit/resources/empty_template_environment.yml
@@ -0,0 +1,9 @@
+channels:
+ - conda-forge
+ - defaults
+dependencies:
+ # This cannot be empty as otherwise no environment will be created.
+ # We're using openssl here as it is available on all system and will
+ # most likely be always installed anyways.
+ # See https://github.com/conda/conda/issues/9487
+ - openssl
diff --git a/pre_commit/resources/empty_template_main.go b/pre_commit/resources/empty_template_main.go
new file mode 100644
index 0000000..38dd16d
--- /dev/null
+++ b/pre_commit/resources/empty_template_main.go
@@ -0,0 +1,3 @@
+package main
+
+func main() {}
diff --git a/pre_commit/resources/empty_template_main.rs b/pre_commit/resources/empty_template_main.rs
new file mode 100644
index 0000000..f328e4d
--- /dev/null
+++ b/pre_commit/resources/empty_template_main.rs
@@ -0,0 +1 @@
+fn main() {}
diff --git a/pre_commit/resources/empty_template_package.json b/pre_commit/resources/empty_template_package.json
new file mode 100644
index 0000000..ac7b725
--- /dev/null
+++ b/pre_commit/resources/empty_template_package.json
@@ -0,0 +1,4 @@
+{
+ "name": "pre_commit_dummy_package",
+ "version": "0.0.0"
+}
diff --git a/pre_commit/resources/empty_template_pre_commit_dummy_package.gemspec b/pre_commit/resources/empty_template_pre_commit_dummy_package.gemspec
new file mode 100644
index 0000000..8bfb40c
--- /dev/null
+++ b/pre_commit/resources/empty_template_pre_commit_dummy_package.gemspec
@@ -0,0 +1,6 @@
+Gem::Specification.new do |s|
+ s.name = 'pre_commit_dummy_package'
+ s.version = '0.0.0'
+ s.summary = 'dummy gem for pre-commit hooks'
+ s.authors = ['Anthony Sottile']
+end
diff --git a/pre_commit/resources/empty_template_setup.py b/pre_commit/resources/empty_template_setup.py
new file mode 100644
index 0000000..6886064
--- /dev/null
+++ b/pre_commit/resources/empty_template_setup.py
@@ -0,0 +1,4 @@
+from setuptools import setup
+
+
+setup(name='pre-commit-dummy-package', version='0.0.0')
diff --git a/pre_commit/resources/hook-tmpl b/pre_commit/resources/hook-tmpl
new file mode 100755
index 0000000..299144e
--- /dev/null
+++ b/pre_commit/resources/hook-tmpl
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# File generated by pre-commit: https://pre-commit.com
+# ID: 138fd403232d2ddd5efb44317e38bf03
+import os
+import sys
+
+# we try our best, but the shebang of this script is difficult to determine:
+# - macos doesn't ship with python3
+# - windows executables are almost always `python.exe`
+# therefore we continue to support python2 for this small script
+if sys.version_info < (3, 3):
+ from distutils.spawn import find_executable as which
+else:
+ from shutil import which
+
+# work around https://github.com/Homebrew/homebrew-core/issues/30445
+os.environ.pop('__PYVENV_LAUNCHER__', None)
+
+# start templated
+INSTALL_PYTHON = ''
+ARGS = ['hook-impl']
+# end templated
+ARGS.extend(('--hook-dir', os.path.realpath(os.path.dirname(__file__))))
+ARGS.append('--')
+ARGS.extend(sys.argv[1:])
+
+DNE = '`pre-commit` not found. Did you forget to activate your virtualenv?'
+if os.access(INSTALL_PYTHON, os.X_OK):
+ CMD = [INSTALL_PYTHON, '-mpre_commit']
+elif which('pre-commit'):
+ CMD = ['pre-commit']
+else:
+ raise SystemExit(DNE)
+
+CMD.extend(ARGS)
+if sys.platform == 'win32': # https://bugs.python.org/issue19124
+ import subprocess
+
+ if sys.version_info < (3, 7): # https://bugs.python.org/issue25942
+ raise SystemExit(subprocess.Popen(CMD).wait())
+ else:
+ raise SystemExit(subprocess.call(CMD))
+else:
+ os.execvp(CMD[0], CMD)
diff --git a/pre_commit/resources/rbenv.tar.gz b/pre_commit/resources/rbenv.tar.gz
new file mode 100644
index 0000000..5307b19
--- /dev/null
+++ b/pre_commit/resources/rbenv.tar.gz
Binary files differ
diff --git a/pre_commit/resources/ruby-build.tar.gz b/pre_commit/resources/ruby-build.tar.gz
new file mode 100644
index 0000000..4a69a09
--- /dev/null
+++ b/pre_commit/resources/ruby-build.tar.gz
Binary files differ
diff --git a/pre_commit/resources/ruby-download.tar.gz b/pre_commit/resources/ruby-download.tar.gz
new file mode 100644
index 0000000..7ccfb6c
--- /dev/null
+++ b/pre_commit/resources/ruby-download.tar.gz
Binary files differ
diff --git a/pre_commit/staged_files_only.py b/pre_commit/staged_files_only.py
new file mode 100644
index 0000000..09d323d
--- /dev/null
+++ b/pre_commit/staged_files_only.py
@@ -0,0 +1,90 @@
+import contextlib
+import logging
+import os.path
+import time
+from typing import Generator
+
+from pre_commit import git
+from pre_commit.util import CalledProcessError
+from pre_commit.util import cmd_output
+from pre_commit.util import cmd_output_b
+from pre_commit.xargs import xargs
+
+
+logger = logging.getLogger('pre_commit')
+
+
+def _git_apply(patch: str) -> None:
+ args = ('apply', '--whitespace=nowarn', patch)
+ try:
+ cmd_output_b('git', *args)
+ except CalledProcessError:
+ # Retry with autocrlf=false -- see #570
+ cmd_output_b('git', '-c', 'core.autocrlf=false', *args)
+
+
+@contextlib.contextmanager
+def _intent_to_add_cleared() -> Generator[None, None, None]:
+ intent_to_add = git.intent_to_add_files()
+ if intent_to_add:
+ logger.warning('Unstaged intent-to-add files detected.')
+
+ xargs(('git', 'rm', '--cached', '--'), intent_to_add)
+ try:
+ yield
+ finally:
+ xargs(('git', 'add', '--intent-to-add', '--'), intent_to_add)
+ else:
+ yield
+
+
+@contextlib.contextmanager
+def _unstaged_changes_cleared(patch_dir: str) -> Generator[None, None, None]:
+ tree = cmd_output('git', 'write-tree')[1].strip()
+ retcode, diff_stdout_binary, _ = cmd_output_b(
+ 'git', 'diff-index', '--ignore-submodules', '--binary',
+ '--exit-code', '--no-color', '--no-ext-diff', tree, '--',
+ retcode=None,
+ )
+ if retcode and diff_stdout_binary.strip():
+ patch_filename = f'patch{int(time.time())}'
+ patch_filename = os.path.join(patch_dir, patch_filename)
+ logger.warning('Unstaged files detected.')
+ logger.info(f'Stashing unstaged files to {patch_filename}.')
+ # Save the current unstaged changes as a patch
+ os.makedirs(patch_dir, exist_ok=True)
+ with open(patch_filename, 'wb') as patch_file:
+ patch_file.write(diff_stdout_binary)
+
+ # Clear the working directory of unstaged changes
+ cmd_output_b('git', 'checkout', '--', '.')
+ try:
+ yield
+ finally:
+ # Try to apply the patch we saved
+ try:
+ _git_apply(patch_filename)
+ except CalledProcessError:
+ logger.warning(
+ 'Stashed changes conflicted with hook auto-fixes... '
+ 'Rolling back fixes...',
+ )
+ # We failed to apply the patch, presumably due to fixes made
+ # by hooks.
+ # Roll back the changes made by hooks.
+ cmd_output_b('git', 'checkout', '--', '.')
+ _git_apply(patch_filename)
+ logger.info(f'Restored changes from {patch_filename}.')
+ else:
+ # There weren't any staged files so we don't need to do anything
+ # special
+ yield
+
+
+@contextlib.contextmanager
+def staged_files_only(patch_dir: str) -> Generator[None, None, None]:
+ """Clear any unstaged changes from the git working directory inside this
+ context.
+ """
+ with _intent_to_add_cleared(), _unstaged_changes_cleared(patch_dir):
+ yield
diff --git a/pre_commit/store.py b/pre_commit/store.py
new file mode 100644
index 0000000..760b37a
--- /dev/null
+++ b/pre_commit/store.py
@@ -0,0 +1,250 @@
+import contextlib
+import logging
+import os.path
+import sqlite3
+import tempfile
+from typing import Callable
+from typing import Generator
+from typing import List
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+
+import pre_commit.constants as C
+from pre_commit import file_lock
+from pre_commit import git
+from pre_commit.util import CalledProcessError
+from pre_commit.util import clean_path_on_failure
+from pre_commit.util import cmd_output_b
+from pre_commit.util import resource_text
+from pre_commit.util import rmtree
+
+
+logger = logging.getLogger('pre_commit')
+
+
+def _get_default_directory() -> str:
+ """Returns the default directory for the Store. This is intentionally
+ underscored to indicate that `Store.get_default_directory` is the intended
+ way to get this information. This is also done so
+ `Store.get_default_directory` can be mocked in tests and
+ `_get_default_directory` can be tested.
+ """
+ return os.environ.get('PRE_COMMIT_HOME') or os.path.join(
+ os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~/.cache'),
+ 'pre-commit',
+ )
+
+
+class Store:
+ get_default_directory = staticmethod(_get_default_directory)
+
+ def __init__(self, directory: Optional[str] = None) -> None:
+ self.directory = directory or Store.get_default_directory()
+ self.db_path = os.path.join(self.directory, 'db.db')
+
+ if not os.path.exists(self.directory):
+ os.makedirs(self.directory, exist_ok=True)
+ with open(os.path.join(self.directory, 'README'), 'w') as f:
+ f.write(
+ 'This directory is maintained by the pre-commit project.\n'
+ 'Learn more: https://github.com/pre-commit/pre-commit\n',
+ )
+
+ if os.path.exists(self.db_path):
+ return
+ with self.exclusive_lock():
+ # Another process may have already completed this work
+ if os.path.exists(self.db_path): # pragma: no cover (race)
+ return
+ # To avoid a race where someone ^Cs between db creation and
+ # execution of the CREATE TABLE statement
+ fd, tmpfile = tempfile.mkstemp(dir=self.directory)
+ # We'll be managing this file ourselves
+ os.close(fd)
+ with self.connect(db_path=tmpfile) as db:
+ db.executescript(
+ 'CREATE TABLE repos ('
+ ' repo TEXT NOT NULL,'
+ ' ref TEXT NOT NULL,'
+ ' path TEXT NOT NULL,'
+ ' PRIMARY KEY (repo, ref)'
+ ');',
+ )
+ self._create_config_table(db)
+
+ # Atomic file move
+ os.rename(tmpfile, self.db_path)
+
+ @contextlib.contextmanager
+ def exclusive_lock(self) -> Generator[None, None, None]:
+ def blocked_cb() -> None: # pragma: no cover (tests are in-process)
+ logger.info('Locking pre-commit directory')
+
+ with file_lock.lock(os.path.join(self.directory, '.lock'), blocked_cb):
+ yield
+
+ @contextlib.contextmanager
+ def connect(
+ self,
+ db_path: Optional[str] = None,
+ ) -> Generator[sqlite3.Connection, None, None]:
+ db_path = db_path or self.db_path
+ # sqlite doesn't close its fd with its contextmanager >.<
+ # contextlib.closing fixes this.
+ # See: https://stackoverflow.com/a/28032829/812183
+ with contextlib.closing(sqlite3.connect(db_path)) as db:
+ # this creates a transaction
+ with db:
+ yield db
+
+ @classmethod
+ def db_repo_name(cls, repo: str, deps: Sequence[str]) -> str:
+ if deps:
+ return f'{repo}:{",".join(sorted(deps))}'
+ else:
+ return repo
+
+ def _new_repo(
+ self,
+ repo: str,
+ ref: str,
+ deps: Sequence[str],
+ make_strategy: Callable[[str], None],
+ ) -> str:
+ repo = self.db_repo_name(repo, deps)
+
+ def _get_result() -> Optional[str]:
+ # Check if we already exist
+ with self.connect() as db:
+ result = db.execute(
+ 'SELECT path FROM repos WHERE repo = ? AND ref = ?',
+ (repo, ref),
+ ).fetchone()
+ return result[0] if result else None
+
+ result = _get_result()
+ if result:
+ return result
+ with self.exclusive_lock():
+ # Another process may have already completed this work
+ result = _get_result()
+ if result: # pragma: no cover (race)
+ return result
+
+ logger.info(f'Initializing environment for {repo}.')
+
+ directory = tempfile.mkdtemp(prefix='repo', dir=self.directory)
+ with clean_path_on_failure(directory):
+ make_strategy(directory)
+
+ # Update our db with the created repo
+ with self.connect() as db:
+ db.execute(
+ 'INSERT INTO repos (repo, ref, path) VALUES (?, ?, ?)',
+ [repo, ref, directory],
+ )
+ return directory
+
+ def _complete_clone(self, ref: str, git_cmd: Callable[..., None]) -> None:
+ """Perform a complete clone of a repository and its submodules """
+
+ git_cmd('fetch', 'origin', '--tags')
+ git_cmd('checkout', ref)
+ git_cmd('submodule', 'update', '--init', '--recursive')
+
+ def _shallow_clone(self, ref: str, git_cmd: Callable[..., None]) -> None:
+ """Perform a shallow clone of a repository and its submodules """
+
+ git_config = 'protocol.version=2'
+ git_cmd('-c', git_config, 'fetch', 'origin', ref, '--depth=1')
+ git_cmd('checkout', 'FETCH_HEAD')
+ git_cmd(
+ '-c', git_config, 'submodule', 'update', '--init', '--recursive',
+ '--depth=1',
+ )
+
+ def clone(self, repo: str, ref: str, deps: Sequence[str] = ()) -> str:
+ """Clone the given url and checkout the specific ref."""
+
+ def clone_strategy(directory: str) -> None:
+ git.init_repo(directory, repo)
+ env = git.no_git_env()
+
+ def _git_cmd(*args: str) -> None:
+ cmd_output_b('git', *args, cwd=directory, env=env)
+
+ try:
+ self._shallow_clone(ref, _git_cmd)
+ except CalledProcessError:
+ self._complete_clone(ref, _git_cmd)
+
+ return self._new_repo(repo, ref, deps, clone_strategy)
+
+ LOCAL_RESOURCES = (
+ 'Cargo.toml', 'main.go', 'main.rs', '.npmignore', 'package.json',
+ 'pre_commit_dummy_package.gemspec', 'setup.py', 'environment.yml',
+ 'Makefile.PL',
+ )
+
+ def make_local(self, deps: Sequence[str]) -> str:
+ def make_local_strategy(directory: str) -> None:
+ for resource in self.LOCAL_RESOURCES:
+ contents = resource_text(f'empty_template_{resource}')
+ with open(os.path.join(directory, resource), 'w') as f:
+ f.write(contents)
+
+ env = git.no_git_env()
+
+ # initialize the git repository so it looks more like cloned repos
+ def _git_cmd(*args: str) -> None:
+ cmd_output_b('git', *args, cwd=directory, env=env)
+
+ git.init_repo(directory, '<<unknown>>')
+ _git_cmd('add', '.')
+ git.commit(repo=directory)
+
+ return self._new_repo(
+ 'local', C.LOCAL_REPO_VERSION, deps, make_local_strategy,
+ )
+
+ def _create_config_table(self, db: sqlite3.Connection) -> None:
+ db.executescript(
+ 'CREATE TABLE IF NOT EXISTS configs ('
+ ' path TEXT NOT NULL,'
+ ' PRIMARY KEY (path)'
+ ');',
+ )
+
+ def mark_config_used(self, path: str) -> None:
+ path = os.path.realpath(path)
+ # don't insert config files that do not exist
+ if not os.path.exists(path):
+ return
+ with self.connect() as db:
+ # TODO: eventually remove this and only create in _create
+ self._create_config_table(db)
+ db.execute('INSERT OR IGNORE INTO configs VALUES (?)', (path,))
+
+ def select_all_configs(self) -> List[str]:
+ with self.connect() as db:
+ self._create_config_table(db)
+ rows = db.execute('SELECT path FROM configs').fetchall()
+ return [path for path, in rows]
+
+ def delete_configs(self, configs: List[str]) -> None:
+ with self.connect() as db:
+ rows = [(path,) for path in configs]
+ db.executemany('DELETE FROM configs WHERE path = ?', rows)
+
+ def select_all_repos(self) -> List[Tuple[str, str, str]]:
+ with self.connect() as db:
+ return db.execute('SELECT repo, ref, path from repos').fetchall()
+
+ def delete_repo(self, db_repo_name: str, ref: str, path: str) -> None:
+ with self.connect() as db:
+ db.execute(
+ 'DELETE FROM repos WHERE repo = ? and ref = ?',
+ (db_repo_name, ref),
+ )
+ rmtree(path)
diff --git a/pre_commit/util.py b/pre_commit/util.py
new file mode 100644
index 0000000..2db579a
--- /dev/null
+++ b/pre_commit/util.py
@@ -0,0 +1,272 @@
+import contextlib
+import errno
+import functools
+import os.path
+import shutil
+import stat
+import subprocess
+import sys
+import tempfile
+from types import TracebackType
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import Generator
+from typing import IO
+from typing import Optional
+from typing import Tuple
+from typing import Type
+from typing import Union
+
+import yaml
+
+from pre_commit import parse_shebang
+
+if sys.version_info >= (3, 7): # pragma: no cover (PY37+)
+ from importlib.resources import open_binary
+ from importlib.resources import read_text
+else: # pragma: no cover (<PY37)
+ from importlib_resources import open_binary
+ from importlib_resources import read_text
+
+EnvironT = Union[Dict[str, str], 'os._Environ']
+
+Loader = getattr(yaml, 'CSafeLoader', yaml.SafeLoader)
+yaml_load = functools.partial(yaml.load, Loader=Loader)
+Dumper = getattr(yaml, 'CSafeDumper', yaml.SafeDumper)
+
+
+def yaml_dump(o: Any) -> str:
+ # when python/mypy#1484 is solved, this can be `functools.partial`
+ return yaml.dump(
+ o, Dumper=Dumper, default_flow_style=False, indent=4, sort_keys=False,
+ )
+
+
+def force_bytes(exc: Any) -> bytes:
+ with contextlib.suppress(TypeError):
+ return bytes(exc)
+ with contextlib.suppress(Exception):
+ return str(exc).encode()
+ return f'<unprintable {type(exc).__name__} object>'.encode()
+
+
+@contextlib.contextmanager
+def clean_path_on_failure(path: str) -> Generator[None, None, None]:
+ """Cleans up the directory on an exceptional failure."""
+ try:
+ yield
+ except BaseException:
+ if os.path.exists(path):
+ rmtree(path)
+ raise
+
+
+@contextlib.contextmanager
+def tmpdir() -> Generator[str, None, None]:
+ """Contextmanager to create a temporary directory. It will be cleaned up
+ afterwards.
+ """
+ tempdir = tempfile.mkdtemp()
+ try:
+ yield tempdir
+ finally:
+ rmtree(tempdir)
+
+
+def resource_bytesio(filename: str) -> IO[bytes]:
+ return open_binary('pre_commit.resources', filename)
+
+
+def resource_text(filename: str) -> str:
+ return read_text('pre_commit.resources', filename)
+
+
+def make_executable(filename: str) -> None:
+ original_mode = os.stat(filename).st_mode
+ new_mode = original_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+ os.chmod(filename, new_mode)
+
+
+class CalledProcessError(RuntimeError):
+ def __init__(
+ self,
+ returncode: int,
+ cmd: Tuple[str, ...],
+ expected_returncode: int,
+ stdout: bytes,
+ stderr: Optional[bytes],
+ ) -> None:
+ super().__init__(returncode, cmd, expected_returncode, stdout, stderr)
+ self.returncode = returncode
+ self.cmd = cmd
+ self.expected_returncode = expected_returncode
+ self.stdout = stdout
+ self.stderr = stderr
+
+ def __bytes__(self) -> bytes:
+ def _indent_or_none(part: Optional[bytes]) -> bytes:
+ if part:
+ return b'\n ' + part.replace(b'\n', b'\n ')
+ else:
+ return b' (none)'
+
+ return b''.join((
+ f'command: {self.cmd!r}\n'.encode(),
+ f'return code: {self.returncode}\n'.encode(),
+ f'expected return code: {self.expected_returncode}\n'.encode(),
+ b'stdout:', _indent_or_none(self.stdout), b'\n',
+ b'stderr:', _indent_or_none(self.stderr),
+ ))
+
+ def __str__(self) -> str:
+ return self.__bytes__().decode()
+
+
+def _setdefault_kwargs(kwargs: Dict[str, Any]) -> None:
+ for arg in ('stdin', 'stdout', 'stderr'):
+ kwargs.setdefault(arg, subprocess.PIPE)
+
+
+def _oserror_to_output(e: OSError) -> Tuple[int, bytes, None]:
+ return 1, force_bytes(e).rstrip(b'\n') + b'\n', None
+
+
+def cmd_output_b(
+ *cmd: str,
+ retcode: Optional[int] = 0,
+ **kwargs: Any,
+) -> Tuple[int, bytes, Optional[bytes]]:
+ _setdefault_kwargs(kwargs)
+
+ try:
+ cmd = parse_shebang.normalize_cmd(cmd)
+ except parse_shebang.ExecutableNotFoundError as e:
+ returncode, stdout_b, stderr_b = e.to_output()
+ else:
+ try:
+ proc = subprocess.Popen(cmd, **kwargs)
+ except OSError as e:
+ returncode, stdout_b, stderr_b = _oserror_to_output(e)
+ else:
+ stdout_b, stderr_b = proc.communicate()
+ returncode = proc.returncode
+
+ if retcode is not None and retcode != returncode:
+ raise CalledProcessError(returncode, cmd, retcode, stdout_b, stderr_b)
+
+ return returncode, stdout_b, stderr_b
+
+
+def cmd_output(*cmd: str, **kwargs: Any) -> Tuple[int, str, Optional[str]]:
+ returncode, stdout_b, stderr_b = cmd_output_b(*cmd, **kwargs)
+ stdout = stdout_b.decode() if stdout_b is not None else None
+ stderr = stderr_b.decode() if stderr_b is not None else None
+ return returncode, stdout, stderr
+
+
+if os.name != 'nt': # pragma: win32 no cover
+ from os import openpty
+ import termios
+
+ class Pty:
+ def __init__(self) -> None:
+ self.r: Optional[int] = None
+ self.w: Optional[int] = None
+
+ def __enter__(self) -> 'Pty':
+ self.r, self.w = openpty()
+
+ # tty flags normally change \n to \r\n
+ attrs = termios.tcgetattr(self.r)
+ assert isinstance(attrs[1], int)
+ attrs[1] &= ~(termios.ONLCR | termios.OPOST)
+ termios.tcsetattr(self.r, termios.TCSANOW, attrs)
+
+ return self
+
+ def close_w(self) -> None:
+ if self.w is not None:
+ os.close(self.w)
+ self.w = None
+
+ def close_r(self) -> None:
+ assert self.r is not None
+ os.close(self.r)
+ self.r = None
+
+ def __exit__(
+ self,
+ exc_type: Optional[Type[BaseException]],
+ exc_value: Optional[BaseException],
+ traceback: Optional[TracebackType],
+ ) -> None:
+ self.close_w()
+ self.close_r()
+
+ def cmd_output_p(
+ *cmd: str,
+ retcode: Optional[int] = 0,
+ **kwargs: Any,
+ ) -> Tuple[int, bytes, Optional[bytes]]:
+ assert retcode is None
+ assert kwargs['stderr'] == subprocess.STDOUT, kwargs['stderr']
+ _setdefault_kwargs(kwargs)
+
+ try:
+ cmd = parse_shebang.normalize_cmd(cmd)
+ except parse_shebang.ExecutableNotFoundError as e:
+ return e.to_output()
+
+ with open(os.devnull) as devnull, Pty() as pty:
+ assert pty.r is not None
+ kwargs.update({'stdin': devnull, 'stdout': pty.w, 'stderr': pty.w})
+ try:
+ proc = subprocess.Popen(cmd, **kwargs)
+ except OSError as e:
+ return _oserror_to_output(e)
+
+ pty.close_w()
+
+ buf = b''
+ while True:
+ try:
+ bts = os.read(pty.r, 4096)
+ except OSError as e:
+ if e.errno == errno.EIO:
+ bts = b''
+ else:
+ raise
+ else:
+ buf += bts
+ if not bts:
+ break
+
+ return proc.wait(), buf, None
+else: # pragma: no cover
+ cmd_output_p = cmd_output_b
+
+
+def rmtree(path: str) -> None:
+ """On windows, rmtree fails for readonly dirs."""
+ def handle_remove_readonly(
+ func: Callable[..., Any],
+ path: str,
+ exc: Tuple[Type[OSError], OSError, TracebackType],
+ ) -> None:
+ excvalue = exc[1]
+ if (
+ func in (os.rmdir, os.remove, os.unlink) and
+ excvalue.errno == errno.EACCES
+ ):
+ for p in (path, os.path.dirname(path)):
+ os.chmod(p, os.stat(p).st_mode | stat.S_IWUSR)
+ func(path)
+ else:
+ raise
+ shutil.rmtree(path, ignore_errors=False, onerror=handle_remove_readonly)
+
+
+def parse_version(s: str) -> Tuple[int, ...]:
+ """poor man's version comparison"""
+ return tuple(int(p) for p in s.split('.'))
diff --git a/pre_commit/xargs.py b/pre_commit/xargs.py
new file mode 100644
index 0000000..5235dc6
--- /dev/null
+++ b/pre_commit/xargs.py
@@ -0,0 +1,157 @@
+import concurrent.futures
+import contextlib
+import math
+import os
+import subprocess
+import sys
+from typing import Any
+from typing import Callable
+from typing import Generator
+from typing import Iterable
+from typing import List
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+from typing import TypeVar
+
+from pre_commit import parse_shebang
+from pre_commit.util import cmd_output_b
+from pre_commit.util import cmd_output_p
+from pre_commit.util import EnvironT
+
+TArg = TypeVar('TArg')
+TRet = TypeVar('TRet')
+
+
+def _environ_size(_env: Optional[EnvironT] = None) -> int:
+ environ = _env if _env is not None else getattr(os, 'environb', os.environ)
+ size = 8 * len(environ) # number of pointers in `envp`
+ for k, v in environ.items():
+ size += len(k) + len(v) + 2 # c strings in `envp`
+ return size
+
+
+def _get_platform_max_length() -> int: # pragma: no cover (platform specific)
+ if os.name == 'posix':
+ maximum = os.sysconf('SC_ARG_MAX') - 2048 - _environ_size()
+ maximum = max(min(maximum, 2 ** 17), 2 ** 12)
+ return maximum
+ elif os.name == 'nt':
+ return 2 ** 15 - 2048 # UNICODE_STRING max - headroom
+ else:
+ # posix minimum
+ return 2 ** 12
+
+
+def _command_length(*cmd: str) -> int:
+ full_cmd = ' '.join(cmd)
+
+ # win32 uses the amount of characters, more details at:
+ # https://github.com/pre-commit/pre-commit/pull/839
+ if sys.platform == 'win32':
+ return len(full_cmd.encode('utf-16le')) // 2
+ else:
+ return len(full_cmd.encode(sys.getfilesystemencoding()))
+
+
+class ArgumentTooLongError(RuntimeError):
+ pass
+
+
+def partition(
+ cmd: Sequence[str],
+ varargs: Sequence[str],
+ target_concurrency: int,
+ _max_length: Optional[int] = None,
+) -> Tuple[Tuple[str, ...], ...]:
+ _max_length = _max_length or _get_platform_max_length()
+
+ # Generally, we try to partition evenly into at least `target_concurrency`
+ # partitions, but we don't want a bunch of tiny partitions.
+ max_args = max(4, math.ceil(len(varargs) / target_concurrency))
+
+ cmd = tuple(cmd)
+ ret = []
+
+ ret_cmd: List[str] = []
+ # Reversed so arguments are in order
+ varargs = list(reversed(varargs))
+
+ total_length = _command_length(*cmd) + 1
+ while varargs:
+ arg = varargs.pop()
+
+ arg_length = _command_length(arg) + 1
+ if (
+ total_length + arg_length <= _max_length and
+ len(ret_cmd) < max_args
+ ):
+ ret_cmd.append(arg)
+ total_length += arg_length
+ elif not ret_cmd:
+ raise ArgumentTooLongError(arg)
+ else:
+ # We've exceeded the length, yield a command
+ ret.append(cmd + tuple(ret_cmd))
+ ret_cmd = []
+ total_length = _command_length(*cmd) + 1
+ varargs.append(arg)
+
+ ret.append(cmd + tuple(ret_cmd))
+
+ return tuple(ret)
+
+
+@contextlib.contextmanager
+def _thread_mapper(maxsize: int) -> Generator[
+ Callable[[Callable[[TArg], TRet], Iterable[TArg]], Iterable[TRet]],
+ None, None,
+]:
+ if maxsize == 1:
+ yield map
+ else:
+ with concurrent.futures.ThreadPoolExecutor(maxsize) as ex:
+ yield ex.map
+
+
+def xargs(
+ cmd: Tuple[str, ...],
+ varargs: Sequence[str],
+ *,
+ color: bool = False,
+ target_concurrency: int = 1,
+ _max_length: int = _get_platform_max_length(),
+ **kwargs: Any,
+) -> Tuple[int, bytes]:
+ """A simplified implementation of xargs.
+
+ color: Make a pty if on a platform that supports it
+ target_concurrency: Target number of partitions to run concurrently
+ """
+ cmd_fn = cmd_output_p if color else cmd_output_b
+ retcode = 0
+ stdout = b''
+
+ try:
+ cmd = parse_shebang.normalize_cmd(cmd)
+ except parse_shebang.ExecutableNotFoundError as e:
+ return e.to_output()[:2]
+
+ partitions = partition(cmd, varargs, target_concurrency, _max_length)
+
+ def run_cmd_partition(
+ run_cmd: Tuple[str, ...],
+ ) -> Tuple[int, bytes, Optional[bytes]]:
+ return cmd_fn(
+ *run_cmd, retcode=None, stderr=subprocess.STDOUT, **kwargs,
+ )
+
+ threads = min(len(partitions), target_concurrency)
+ with _thread_mapper(threads) as thread_map:
+ results = thread_map(run_cmd_partition, partitions)
+
+ for proc_retcode, proc_out, _ in results:
+ retcode = max(retcode, proc_retcode)
+ stdout += proc_out
+
+ return retcode, stdout