Merging upstream version 2.5.1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

6 files changed, 88 insertions, 117 deletions

diff --git a/pre_commit/languages/all.py b/pre_commit/languages/all.py
index 8f4ffa8..5609631 100644
--- a/pre_commit/languages/all.py
+++ b/pre_commit/languages/all.py
@@ -14,7 +14,6 @@ from pre_commit.languages import node
 from pre_commit.languages import perl
 from pre_commit.languages import pygrep
 from pre_commit.languages import python
-from pre_commit.languages import python_venv
 from pre_commit.languages import ruby
 from pre_commit.languages import rust
 from pre_commit.languages import script
@@ -49,7 +48,6 @@ languages = {
     'perl': Language(name='perl', ENVIRONMENT_DIR=perl.ENVIRONMENT_DIR, get_default_version=perl.get_default_version, healthy=perl.healthy, install_environment=perl.install_environment, run_hook=perl.run_hook),  # noqa: E501
     'pygrep': Language(name='pygrep', ENVIRONMENT_DIR=pygrep.ENVIRONMENT_DIR, get_default_version=pygrep.get_default_version, healthy=pygrep.healthy, install_environment=pygrep.install_environment, run_hook=pygrep.run_hook),  # noqa: E501
     'python': Language(name='python', ENVIRONMENT_DIR=python.ENVIRONMENT_DIR, get_default_version=python.get_default_version, healthy=python.healthy, install_environment=python.install_environment, run_hook=python.run_hook),  # noqa: E501
-    'python_venv': Language(name='python_venv', ENVIRONMENT_DIR=python_venv.ENVIRONMENT_DIR, get_default_version=python_venv.get_default_version, healthy=python_venv.healthy, install_environment=python_venv.install_environment, run_hook=python_venv.run_hook),  # noqa: E501
     'ruby': Language(name='ruby', ENVIRONMENT_DIR=ruby.ENVIRONMENT_DIR, get_default_version=ruby.get_default_version, healthy=ruby.healthy, install_environment=ruby.install_environment, run_hook=ruby.run_hook),  # noqa: E501
     'rust': Language(name='rust', ENVIRONMENT_DIR=rust.ENVIRONMENT_DIR, get_default_version=rust.get_default_version, healthy=rust.healthy, install_environment=rust.install_environment, run_hook=rust.run_hook),  # noqa: E501
     'script': Language(name='script', ENVIRONMENT_DIR=script.ENVIRONMENT_DIR, get_default_version=script.get_default_version, healthy=script.healthy, install_environment=script.install_environment, run_hook=script.run_hook),  # noqa: E501
@@ -57,4 +55,6 @@ languages = {
     'system': Language(name='system', ENVIRONMENT_DIR=system.ENVIRONMENT_DIR, get_default_version=system.get_default_version, healthy=system.healthy, install_environment=system.install_environment, run_hook=system.run_hook),  # noqa: E501
     # END GENERATED
 }
+# TODO: fully deprecate `python_venv`
+languages['python_venv'] = languages['python']
 all_languages = sorted(languages)
diff --git a/pre_commit/languages/helpers.py b/pre_commit/languages/helpers.py
index b5c95e5..01c65ab 100644
--- a/pre_commit/languages/helpers.py
+++ b/pre_commit/languages/helpers.py
@@ -18,7 +18,7 @@ from pre_commit.xargs import xargs
 if TYPE_CHECKING:
     from typing import NoReturn
 
-FIXED_RANDOM_SEED = 1542676186
+FIXED_RANDOM_SEED = 1542676187
 
 
 def run_setup_cmd(prefix: Prefix, cmd: Tuple[str, ...]) -> None:
@@ -92,7 +92,7 @@ def _shuffled(seq: Sequence[str]) -> List[str]:
     fixed_random.seed(FIXED_RANDOM_SEED, version=1)
 
     seq = list(seq)
-    random.shuffle(seq, random=fixed_random.random)
+    fixed_random.shuffle(seq)
     return seq
 
 
diff --git a/pre_commit/languages/node.py b/pre_commit/languages/node.py
index 9b636d3..26f4919 100644
--- a/pre_commit/languages/node.py
+++ b/pre_commit/languages/node.py
@@ -79,7 +79,7 @@ def install_environment(
 
     # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx?f=255&MSPPError=-2147217396#maxpath
     if sys.platform == 'win32':  # pragma: no cover
-        envdir = f'\\\\?\\{os.path.normpath(envdir)}'
+        envdir = fr'\\?\{os.path.normpath(envdir)}'
     with clean_path_on_failure(envdir):
         cmd = [
             sys.executable, '-mnodeenv', '--prebuilt', '--clean-src', envdir,
diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py
index 85d8281..6f7c900 100644
--- a/pre_commit/languages/python.py
+++ b/pre_commit/languages/python.py
@@ -2,8 +2,7 @@ import contextlib
 import functools
 import os
 import sys
-from typing import Callable
-from typing import ContextManager
+from typing import Dict
 from typing import Generator
 from typing import Optional
 from typing import Sequence
@@ -26,6 +25,28 @@ from pre_commit.util import cmd_output_b
 ENVIRONMENT_DIR = 'py_env'
 
 
+@functools.lru_cache(maxsize=None)
+def _version_info(exe: str) -> str:
+    prog = 'import sys;print(".".join(str(p) for p in sys.version_info))'
+    try:
+        return cmd_output(exe, '-S', '-c', prog)[1].strip()
+    except CalledProcessError:
+        return f'<<error retrieving version from {exe}>>'
+
+
+def _read_pyvenv_cfg(filename: str) -> Dict[str, str]:
+    ret = {}
+    with open(filename) as f:
+        for line in f:
+            try:
+                k, v = line.split('=')
+            except ValueError:  # blank line / comment / etc.
+                continue
+            else:
+                ret[k.strip()] = v.strip()
+    return ret
+
+
 def bin_dir(venv: str) -> str:
     """On windows there's a different directory for the virtualenv"""
     bin_part = 'Scripts' if os.name == 'nt' else 'bin'
@@ -34,6 +55,7 @@ def bin_dir(venv: str) -> str:
 
 def get_env_patch(venv: str) -> PatchesT:
     return (
+        ('PIP_DISABLE_PIP_VERSION_CHECK', '1'),
         ('PYTHONHOME', UNSET),
         ('VIRTUAL_ENV', venv),
         ('PATH', (bin_dir(venv), os.pathsep, Var('PATH'))),
@@ -45,9 +67,10 @@ def _find_by_py_launcher(
 ) -> Optional[str]:  # pragma: no cover (windows only)
     if version.startswith('python'):
         num = version[len('python'):]
+        cmd = ('py', f'-{num}', '-c', 'import sys; print(sys.executable)')
+        env = dict(os.environ, PYTHONIOENCODING='UTF-8')
         try:
-            cmd = ('py', f'-{num}', '-c', 'import sys; print(sys.executable)')
-            return cmd_output(*cmd)[1].strip()
+            return cmd_output(*cmd, env=env)[1].strip()
         except CalledProcessError:
             pass
     return None
@@ -115,6 +138,9 @@ def _sys_executable_matches(version: str) -> bool:
 
 
 def norm_version(version: str) -> str:
+    if version == C.DEFAULT:
+        return os.path.realpath(sys.executable)
+
     # first see if our current executable is appropriate
     if _sys_executable_matches(version):
         return sys.executable
@@ -139,70 +165,59 @@ def norm_version(version: str) -> str:
     return os.path.expanduser(version)
 
 
-def py_interface(
-        _dir: str,
-        _make_venv: Callable[[str, str], None],
-) -> Tuple[
-    Callable[[Prefix, str], ContextManager[None]],
-    Callable[[Prefix, str], bool],
-    Callable[[Hook, Sequence[str], bool], Tuple[int, bytes]],
-    Callable[[Prefix, str, Sequence[str]], None],
-]:
-    @contextlib.contextmanager
-    def in_env(
-            prefix: Prefix,
-            language_version: str,
-    ) -> Generator[None, None, None]:
-        envdir = prefix.path(helpers.environment_dir(_dir, language_version))
-        with envcontext(get_env_patch(envdir)):
-            yield
-
-    def healthy(prefix: Prefix, language_version: str) -> bool:
-        envdir = helpers.environment_dir(_dir, language_version)
-        exe_name = 'python.exe' if sys.platform == 'win32' else 'python'
-        py_exe = prefix.path(bin_dir(envdir), exe_name)
-        with in_env(prefix, language_version):
-            retcode, _, _ = cmd_output_b(
-                py_exe, '-c', 'import ctypes, datetime, io, os, ssl, weakref',
-                cwd='/',
-                retcode=None,
-            )
-        return retcode == 0
-
-    def run_hook(
-            hook: Hook,
-            file_args: Sequence[str],
-            color: bool,
-    ) -> Tuple[int, bytes]:
-        with in_env(hook.prefix, hook.language_version):
-            return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
-
-    def install_environment(
-            prefix: Prefix,
-            version: str,
-            additional_dependencies: Sequence[str],
-    ) -> None:
-        directory = helpers.environment_dir(_dir, version)
-        install = ('python', '-mpip', 'install', '.', *additional_dependencies)
-
-        env_dir = prefix.path(directory)
-        with clean_path_on_failure(env_dir):
-            if version != C.DEFAULT:
-                python = norm_version(version)
-            else:
-                python = os.path.realpath(sys.executable)
-            _make_venv(env_dir, python)
-            with in_env(prefix, version):
-                helpers.run_setup_cmd(prefix, install)
+@contextlib.contextmanager
+def in_env(
+        prefix: Prefix,
+        language_version: str,
+) -> Generator[None, None, None]:
+    directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
+    envdir = prefix.path(directory)
+    with envcontext(get_env_patch(envdir)):
+        yield
 
-    return in_env, healthy, run_hook, install_environment
 
+def healthy(prefix: Prefix, language_version: str) -> bool:
+    directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
+    envdir = prefix.path(directory)
+    pyvenv_cfg = os.path.join(envdir, 'pyvenv.cfg')
 
-def make_venv(envdir: str, python: str) -> None:
-    env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1')
-    cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python)
-    cmd_output_b(*cmd, env=env, cwd='/')
+    # created with "old" virtualenv
+    if not os.path.exists(pyvenv_cfg):
+        return False
+
+    exe_name = 'python.exe' if sys.platform == 'win32' else 'python'
+    py_exe = prefix.path(bin_dir(envdir), exe_name)
+    cfg = _read_pyvenv_cfg(pyvenv_cfg)
+
+    return (
+        'version_info' in cfg and
+        _version_info(py_exe) == cfg['version_info'] and (
+            'base-executable' not in cfg or
+            _version_info(cfg['base-executable']) == cfg['version_info']
+        )
+    )
 
 
-_interface = py_interface(ENVIRONMENT_DIR, make_venv)
-in_env, healthy, run_hook, install_environment = _interface
+def install_environment(
+        prefix: Prefix,
+        version: str,
+        additional_dependencies: Sequence[str],
+) -> None:
+    envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
+    python = norm_version(version)
+    venv_cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python)
+    install_cmd = ('python', '-mpip', 'install', '.', *additional_dependencies)
+
+    with clean_path_on_failure(envdir):
+        cmd_output_b(*venv_cmd, cwd='/')
+        with in_env(prefix, version):
+            helpers.run_setup_cmd(prefix, install_cmd)
+
+
+def run_hook(
+        hook: Hook,
+        file_args: Sequence[str],
+        color: bool,
+) -> Tuple[int, bytes]:
+    with in_env(hook.prefix, hook.language_version):
+        return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
diff --git a/pre_commit/languages/python_venv.py b/pre_commit/languages/python_venv.py
deleted file mode 100644
index 5404c8b..0000000
--- a/pre_commit/languages/python_venv.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os.path
-
-from pre_commit.languages import python
-from pre_commit.util import CalledProcessError
-from pre_commit.util import cmd_output
-from pre_commit.util import cmd_output_b
-
-ENVIRONMENT_DIR = 'py_venv'
-get_default_version = python.get_default_version
-
-
-def orig_py_exe(exe: str) -> str:  # pragma: no cover (platform specific)
-    """A -mvenv virtualenv made from a -mvirtualenv virtualenv installs
-    packages to the incorrect location.  Attempt to find the _original_ exe
-    and invoke `-mvenv` from there.
-
-    See:
-    - https://github.com/pre-commit/pre-commit/issues/755
-    - https://github.com/pypa/virtualenv/issues/1095
-    - https://bugs.python.org/issue30811
-    """
-    try:
-        prefix_script = 'import sys; print(sys.real_prefix)'
-        _, prefix, _ = cmd_output(exe, '-c', prefix_script)
-        prefix = prefix.strip()
-    except CalledProcessError:
-        # not created from -mvirtualenv
-        return exe
-
-    if os.name == 'nt':
-        expected = os.path.join(prefix, 'python.exe')
-    else:
-        expected = os.path.join(prefix, 'bin', os.path.basename(exe))
-
-    if os.path.exists(expected):
-        return expected
-    else:
-        return exe
-
-
-def make_venv(envdir: str, python: str) -> None:
-    cmd_output_b(orig_py_exe(python), '-mvenv', envdir, cwd='/')
-
-
-_interface = python.py_interface(ENVIRONMENT_DIR, make_venv)
-in_env, healthy, run_hook, install_environment = _interface
diff --git a/pre_commit/languages/ruby.py b/pre_commit/languages/ruby.py
index 61241f8..fe524ec 100644
--- a/pre_commit/languages/ruby.py
+++ b/pre_commit/languages/ruby.py
@@ -9,6 +9,7 @@ from typing import Tuple
 import pre_commit.constants as C
 from pre_commit.envcontext import envcontext
 from pre_commit.envcontext import PatchesT
+from pre_commit.envcontext import UNSET
 from pre_commit.envcontext import Var
 from pre_commit.hook import Hook
 from pre_commit.languages import helpers
@@ -28,6 +29,7 @@ def get_env_patch(
 ) -> PatchesT:  # pragma: win32 no cover
     patches: PatchesT = (
         ('GEM_HOME', os.path.join(venv, 'gems')),
+        ('GEM_PATH', UNSET),
         ('RBENV_ROOT', venv),
         ('BUNDLE_IGNORE_CONFIG', '1'),
         (