Repository: pre-commit/pre-commit

Issue:
Most pre-commit hooks broken after OS upgrade to 20.04

```
repos:
-   repo: https://github.com/pre-commit/pre-commit-hooks
    rev: v2.5.0
    hooks:
    - id: check-ast
```

```
ahonnecke@homonym:~/src/web3/web3-maker-scraper$ pre-commit
Check python ast.........................................................Failed
- hook id: check-ast
- exit code: 1

Traceback (most recent call last):
  File "/home/ahonnecke/.cache/pre-commit/repoliux_i6a/py_env-python3/bin/check-ast", line 5, in <module>
    from pre_commit_hooks.check_ast import main
ModuleNotFoundError: No module named 'pre_commit_hooks'
```
I get the same behavior from all my hooks (except pyupgrade)

```
Traceback (most recent call last):
  File "/home/ahonnecke/.cache/pre-commit/repozqy6dpbu/py_env-python3/bin/black", line 5, in <module>
    from black import patched_main
ModuleNotFoundError: No module named 'black'
```
I tried uninstalling and reinstalling all those pip packages (both globally and with --user):
`sudo pip install pylint pytest flake8 mypy pre-commit isort black`


Discussion:
that's strange, it worked fine for me when I upgrade from 18.04 -> 20.04 -- pre-commit has code to detect this and recreate environments even

can you poke around in `/home/ahonnecke/.cache/pre-commit/repozqy6dpbu` and see if things got (partially) deleted?  usually when upgrading operating systems virtualenvs need rebuilding
an `ls -al` in that folder + `ls -al` in that folder + `py_env-python3/bin` will probably tell me enough information
and likely `pre-commit clean` will put you back into a working state, but I do want to figure out what's wrong and if there's anything more that I can do to fix it
oh I know what happened, yeah I'll work on fixing this

the python version changed, but since virtualenv 20.x the executable is just a symlink -- so the symlink was valid and the "healthy" check succeeded but site-packages was at a completely different path resulting in import errors
My apologies, I didn't look at this before I tried blowing away the cache, removing it resolved it:
`ahonnecke@homonym:~/.cache$ rm -rf pre-commit`
and then
`pre-commit run --all-files`

Thanks!
I did list the .cache directory if that's of any help:
`drwxr-xr-x 12 ahonnecke ahonnecke  4096 May  2 13:07 pre-commit/`
ok I'm able to reproduce this in docker, but only after doing a strange thing to teh system python -- but in theory it seems possible to end up in this case in some other way

```dockerfile
FROM ubuntu:bionic AS builder
RUN : \
    && apt-get update \
    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
        ca-certificates \
        curl \
        git \
        python3 \
        python3-distutils \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*

# I couldn't figure out how to force virtualenv to symlink to python3
# so this hack works :shrugs:
RUN mv /usr/bin/python3.6 /usr/bin/python3

ENV PATH=/venv/bin:$PATH
RUN : \
    && curl https://asottile.github.io/get-virtualenv.py | python3 - /venv \
    && pip install pre-commit

RUN : \
    && git clone https://github.com/asottile/astpretty /astpretty \
    && cd /astpretty \
    && echo 'default_language_version:' >> .pre-commit-config.yaml \
    && echo '  python: python3' >> .pre-commit-config.yaml \
    && pre-commit run --all-files

FROM ubuntu:focal AS final
RUN : \
    && apt-get update \
    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
        ca-certificates \
        curl \
        gcc \
        g++ \
        git \
        python3-dev \
        python3-distutils \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*

ENV PATH=/venv/bin:$PATH
RUN : \
    && curl https://asottile.github.io/get-virtualenv.py | python3 - /venv \
    && pip install pre-commit

COPY --from=builder /root/.cache/pre-commit /root/.cache/pre-commit
COPY --from=builder /astpretty /astpretty
RUN cd /astpretty && pre-commit run --all-files
```

```console
$ docker build -t test .

...

setup-cfg-fmt............................................................Failed
- hook id: setup-cfg-fmt
- exit code: 1

Traceback (most recent call last):
  File "/root/.cache/pre-commit/repo_vb9lwwz/py_env-python3/bin/setup-cfg-fmt", line 5, in <module>
    from setup_cfg_fmt import main
ModuleNotFoundError: No module named 'setup_cfg_fmt'

mypy.....................................................................Failed
- hook id: mypy
- exit code: 1

Traceback (most recent call last):
  File "/root/.cache/pre-commit/repokpzyv_r7/py_env-python3/bin/mypy", line 5, in <module>
    from mypy.__main__ import console_entry
ModuleNotFoundError: No module named 'mypy'

The command '/bin/sh -c cd /astpretty && pre-commit run --all-files' returned a non-zero code: 1
```

Resolving patch:
diff --git a/pre_commit/languages/all.py b/pre_commit/languages/all.py
--- a/pre_commit/languages/all.py
+++ b/pre_commit/languages/all.py
@@ -14,7 +14,6 @@
 from pre_commit.languages import perl
 from pre_commit.languages import pygrep
 from pre_commit.languages import python
-from pre_commit.languages import python_venv
 from pre_commit.languages import ruby
 from pre_commit.languages import rust
 from pre_commit.languages import script
@@ -49,7 +48,6 @@ class Language(NamedTuple):
     'perl': Language(name='perl', ENVIRONMENT_DIR=perl.ENVIRONMENT_DIR, get_default_version=perl.get_default_version, healthy=perl.healthy, install_environment=perl.install_environment, run_hook=perl.run_hook),  # noqa: E501
     'pygrep': Language(name='pygrep', ENVIRONMENT_DIR=pygrep.ENVIRONMENT_DIR, get_default_version=pygrep.get_default_version, healthy=pygrep.healthy, install_environment=pygrep.install_environment, run_hook=pygrep.run_hook),  # noqa: E501
     'python': Language(name='python', ENVIRONMENT_DIR=python.ENVIRONMENT_DIR, get_default_version=python.get_default_version, healthy=python.healthy, install_environment=python.install_environment, run_hook=python.run_hook),  # noqa: E501
-    'python_venv': Language(name='python_venv', ENVIRONMENT_DIR=python_venv.ENVIRONMENT_DIR, get_default_version=python_venv.get_default_version, healthy=python_venv.healthy, install_environment=python_venv.install_environment, run_hook=python_venv.run_hook),  # noqa: E501
     'ruby': Language(name='ruby', ENVIRONMENT_DIR=ruby.ENVIRONMENT_DIR, get_default_version=ruby.get_default_version, healthy=ruby.healthy, install_environment=ruby.install_environment, run_hook=ruby.run_hook),  # noqa: E501
     'rust': Language(name='rust', ENVIRONMENT_DIR=rust.ENVIRONMENT_DIR, get_default_version=rust.get_default_version, healthy=rust.healthy, install_environment=rust.install_environment, run_hook=rust.run_hook),  # noqa: E501
     'script': Language(name='script', ENVIRONMENT_DIR=script.ENVIRONMENT_DIR, get_default_version=script.get_default_version, healthy=script.healthy, install_environment=script.install_environment, run_hook=script.run_hook),  # noqa: E501
@@ -57,4 +55,6 @@ class Language(NamedTuple):
     'system': Language(name='system', ENVIRONMENT_DIR=system.ENVIRONMENT_DIR, get_default_version=system.get_default_version, healthy=system.healthy, install_environment=system.install_environment, run_hook=system.run_hook),  # noqa: E501
     # END GENERATED
 }
+# TODO: fully deprecate `python_venv`
+languages['python_venv'] = languages['python']
 all_languages = sorted(languages)
diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py
--- a/pre_commit/languages/python.py
+++ b/pre_commit/languages/python.py
@@ -2,8 +2,7 @@
 import functools
 import os
 import sys
-from typing import Callable
-from typing import ContextManager
+from typing import Dict
 from typing import Generator
 from typing import Optional
 from typing import Sequence
@@ -26,6 +25,28 @@
 ENVIRONMENT_DIR = 'py_env'
 
 
+@functools.lru_cache(maxsize=None)
+def _version_info(exe: str) -> str:
+    prog = 'import sys;print(".".join(str(p) for p in sys.version_info))'
+    try:
+        return cmd_output(exe, '-S', '-c', prog)[1].strip()
+    except CalledProcessError:
+        return f'<<error retrieving version from {exe}>>'
+
+
+def _read_pyvenv_cfg(filename: str) -> Dict[str, str]:
+    ret = {}
+    with open(filename) as f:
+        for line in f:
+            try:
+                k, v = line.split('=')
+            except ValueError:  # blank line / comment / etc.
+                continue
+            else:
+                ret[k.strip()] = v.strip()
+    return ret
+
+
 def bin_dir(venv: str) -> str:
     """On windows there's a different directory for the virtualenv"""
     bin_part = 'Scripts' if os.name == 'nt' else 'bin'
@@ -116,6 +137,9 @@ def _sys_executable_matches(version: str) -> bool:
 
 
 def norm_version(version: str) -> str:
+    if version == C.DEFAULT:
+        return os.path.realpath(sys.executable)
+
     # first see if our current executable is appropriate
     if _sys_executable_matches(version):
         return sys.executable
@@ -140,70 +164,59 @@ def norm_version(version: str) -> str:
     return os.path.expanduser(version)
 
 
-def py_interface(
-        _dir: str,
-        _make_venv: Callable[[str, str], None],
-) -> Tuple[
-    Callable[[Prefix, str], ContextManager[None]],
-    Callable[[Prefix, str], bool],
-    Callable[[Hook, Sequence[str], bool], Tuple[int, bytes]],
-    Callable[[Prefix, str, Sequence[str]], None],
-]:
-    @contextlib.contextmanager
-    def in_env(
-            prefix: Prefix,
-            language_version: str,
-    ) -> Generator[None, None, None]:
-        envdir = prefix.path(helpers.environment_dir(_dir, language_version))
-        with envcontext(get_env_patch(envdir)):
-            yield
-
-    def healthy(prefix: Prefix, language_version: str) -> bool:
-        envdir = helpers.environment_dir(_dir, language_version)
-        exe_name = 'python.exe' if sys.platform == 'win32' else 'python'
-        py_exe = prefix.path(bin_dir(envdir), exe_name)
-        with in_env(prefix, language_version):
-            retcode, _, _ = cmd_output_b(
-                py_exe, '-c', 'import ctypes, datetime, io, os, ssl, weakref',
-                cwd='/',
-                retcode=None,
-            )
-        return retcode == 0
-
-    def run_hook(
-            hook: Hook,
-            file_args: Sequence[str],
-            color: bool,
-    ) -> Tuple[int, bytes]:
-        with in_env(hook.prefix, hook.language_version):
-            return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
-
-    def install_environment(
-            prefix: Prefix,
-            version: str,
-            additional_dependencies: Sequence[str],
-    ) -> None:
-        directory = helpers.environment_dir(_dir, version)
-        install = ('python', '-mpip', 'install', '.', *additional_dependencies)
-
-        env_dir = prefix.path(directory)
-        with clean_path_on_failure(env_dir):
-            if version != C.DEFAULT:
-                python = norm_version(version)
-            else:
-                python = os.path.realpath(sys.executable)
-            _make_venv(env_dir, python)
-            with in_env(prefix, version):
-                helpers.run_setup_cmd(prefix, install)
+@contextlib.contextmanager
+def in_env(
+        prefix: Prefix,
+        language_version: str,
+) -> Generator[None, None, None]:
+    directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
+    envdir = prefix.path(directory)
+    with envcontext(get_env_patch(envdir)):
+        yield
 
-    return in_env, healthy, run_hook, install_environment
 
+def healthy(prefix: Prefix, language_version: str) -> bool:
+    directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
+    envdir = prefix.path(directory)
+    pyvenv_cfg = os.path.join(envdir, 'pyvenv.cfg')
 
-def make_venv(envdir: str, python: str) -> None:
-    env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1')
-    cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python)
-    cmd_output_b(*cmd, env=env, cwd='/')
+    # created with "old" virtualenv
+    if not os.path.exists(pyvenv_cfg):
+        return False
+
+    exe_name = 'python.exe' if sys.platform == 'win32' else 'python'
+    py_exe = prefix.path(bin_dir(envdir), exe_name)
+    cfg = _read_pyvenv_cfg(pyvenv_cfg)
+
+    return (
+        'version_info' in cfg and
+        _version_info(py_exe) == cfg['version_info'] and (
+            'base-executable' not in cfg or
+            _version_info(cfg['base-executable']) == cfg['version_info']
+        )
+    )
 
 
-_interface = py_interface(ENVIRONMENT_DIR, make_venv)
-in_env, healthy, run_hook, install_environment = _interface
+def install_environment(
+        prefix: Prefix,
+        version: str,
+        additional_dependencies: Sequence[str],
+) -> None:
+    envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
+    python = norm_version(version)
+    venv_cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python)
+    install_cmd = ('python', '-mpip', 'install', '.', *additional_dependencies)
+
+    with clean_path_on_failure(envdir):
+        cmd_output_b(*venv_cmd, cwd='/')
+        with in_env(prefix, version):
+            helpers.run_setup_cmd(prefix, install_cmd)
+
+
+def run_hook(
+        hook: Hook,
+        file_args: Sequence[str],
+        color: bool,
+) -> Tuple[int, bytes]:
+    with in_env(hook.prefix, hook.language_version):
+        return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
diff --git a/pre_commit/languages/python_venv.py b/pre_commit/languages/python_venv.py
deleted file mode 100644
--- a/pre_commit/languages/python_venv.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os.path
-
-from pre_commit.languages import python
-from pre_commit.util import CalledProcessError
-from pre_commit.util import cmd_output
-from pre_commit.util import cmd_output_b
-
-ENVIRONMENT_DIR = 'py_venv'
-get_default_version = python.get_default_version
-
-
-def orig_py_exe(exe: str) -> str:  # pragma: no cover (platform specific)
-    """A -mvenv virtualenv made from a -mvirtualenv virtualenv installs
-    packages to the incorrect location.  Attempt to find the _original_ exe
-    and invoke `-mvenv` from there.
-
-    See:
-    - https://github.com/pre-commit/pre-commit/issues/755
-    - https://github.com/pypa/virtualenv/issues/1095
-    - https://bugs.python.org/issue30811
-    """
-    try:
-        prefix_script = 'import sys; print(sys.real_prefix)'
-        _, prefix, _ = cmd_output(exe, '-c', prefix_script)
-        prefix = prefix.strip()
-    except CalledProcessError:
-        # not created from -mvirtualenv
-        return exe
-
-    if os.name == 'nt':
-        expected = os.path.join(prefix, 'python.exe')
-    else:
-        expected = os.path.join(prefix, 'bin', os.path.basename(exe))
-
-    if os.path.exists(expected):
-        return expected
-    else:
-        return exe
-
-
-def make_venv(envdir: str, python: str) -> None:
-    cmd_output_b(orig_py_exe(python), '-mvenv', envdir, cwd='/')
-
-
-_interface = python.py_interface(ENVIRONMENT_DIR, make_venv)
-in_env, healthy, run_hook, install_environment = _interface


Test patch:
diff --git a/testing/gen-languages-all b/testing/gen-languages-all
--- a/testing/gen-languages-all
+++ b/testing/gen-languages-all
@@ -3,8 +3,7 @@ import sys
 
 LANGUAGES = [
     'conda', 'docker', 'docker_image', 'fail', 'golang', 'node', 'perl',
-    'pygrep', 'python', 'python_venv', 'ruby', 'rust', 'script', 'swift',
-    'system',
+    'pygrep', 'python', 'ruby', 'rust', 'script', 'swift', 'system',
 ]
 FIELDS = [
     'ENVIRONMENT_DIR', 'get_default_version', 'healthy', 'install_environment',
diff --git a/testing/util.py b/testing/util.py
--- a/testing/util.py
+++ b/testing/util.py
@@ -45,20 +45,6 @@ def cmd_output_mocked_pre_commit_home(
 xfailif_windows = pytest.mark.xfail(os.name == 'nt', reason='windows')
 
 
-def supports_venv():  # pragma: no cover (platform specific)
-    try:
-        __import__('ensurepip')
-        __import__('venv')
-        return True
-    except ImportError:
-        return False
-
-
-xfailif_no_venv = pytest.mark.xfail(
-    not supports_venv(), reason='Does not support venv module',
-)
-
-
 def run_opts(
         all_files=False,
         files=(),
diff --git a/tests/languages/python_test.py b/tests/languages/python_test.py
--- a/tests/languages/python_test.py
+++ b/tests/languages/python_test.py
@@ -5,10 +5,23 @@
 import pytest
 
 import pre_commit.constants as C
+from pre_commit.envcontext import envcontext
 from pre_commit.languages import python
 from pre_commit.prefix import Prefix
 
 
+def test_read_pyvenv_cfg(tmpdir):
+    pyvenv_cfg = tmpdir.join('pyvenv.cfg')
+    pyvenv_cfg.write(
+        '# I am a comment\n'
+        '\n'
+        'foo = bar\n'
+        'version-info=123\n',
+    )
+    expected = {'foo': 'bar', 'version-info': '123'}
+    assert python._read_pyvenv_cfg(pyvenv_cfg) == expected
+
+
 def test_norm_version_expanduser():
     home = os.path.expanduser('~')
     if os.name == 'nt':  # pragma: nt cover
@@ -21,6 +34,10 @@ def test_norm_version_expanduser():
     assert result == expected_path
 
 
+def test_norm_version_of_default_is_sys_executable():
+    assert python.norm_version('default') == os.path.realpath(sys.executable)
+
+
 @pytest.mark.parametrize('v', ('python3.6', 'python3', 'python'))
 def test_sys_executable_matches(v):
     with mock.patch.object(sys, 'version_info', (3, 6, 7)):
@@ -49,27 +66,78 @@ def test_find_by_sys_executable(exe, realpath, expected):
                 assert python._find_by_sys_executable() == expected
 
 
-def test_healthy_types_py_in_cwd(tmpdir):
+@pytest.fixture
+def python_dir(tmpdir):
     with tmpdir.as_cwd():
         prefix = tmpdir.join('prefix').ensure_dir()
         prefix.join('setup.py').write('import setuptools; setuptools.setup()')
         prefix = Prefix(str(prefix))
-        python.install_environment(prefix, C.DEFAULT, ())
+        yield prefix, tmpdir
 
-        # even if a `types.py` file exists, should still be healthy
-        tmpdir.join('types.py').ensure()
-        assert python.healthy(prefix, C.DEFAULT) is True
 
+def test_healthy_default_creator(python_dir):
+    prefix, tmpdir = python_dir
 
-def test_healthy_python_goes_missing(tmpdir):
-    with tmpdir.as_cwd():
-        prefix = tmpdir.join('prefix').ensure_dir()
-        prefix.join('setup.py').write('import setuptools; setuptools.setup()')
-        prefix = Prefix(str(prefix))
+    python.install_environment(prefix, C.DEFAULT, ())
+
+    # should be healthy right after creation
+    assert python.healthy(prefix, C.DEFAULT) is True
+
+    # even if a `types.py` file exists, should still be healthy
+    tmpdir.join('types.py').ensure()
+    assert python.healthy(prefix, C.DEFAULT) is True
+
+
+def test_healthy_venv_creator(python_dir):
+    # venv creator produces slightly different pyvenv.cfg
+    prefix, tmpdir = python_dir
+
+    with envcontext((('VIRTUALENV_CREATOR', 'venv'),)):
         python.install_environment(prefix, C.DEFAULT, ())
 
-        exe_name = 'python' if sys.platform != 'win32' else 'python.exe'
-        py_exe = prefix.path(python.bin_dir('py_env-default'), exe_name)
-        os.remove(py_exe)
+    assert python.healthy(prefix, C.DEFAULT) is True
+
+
+def test_unhealthy_python_goes_missing(python_dir):
+    prefix, tmpdir = python_dir
+
+    python.install_environment(prefix, C.DEFAULT, ())
+
+    exe_name = 'python' if sys.platform != 'win32' else 'python.exe'
+    py_exe = prefix.path(python.bin_dir('py_env-default'), exe_name)
+    os.remove(py_exe)
+
+    assert python.healthy(prefix, C.DEFAULT) is False
+
+
+def test_unhealthy_with_version_change(python_dir):
+    prefix, tmpdir = python_dir
+
+    python.install_environment(prefix, C.DEFAULT, ())
+
+    with open(prefix.path('py_env-default/pyvenv.cfg'), 'w') as f:
+        f.write('version_info = 1.2.3\n')
+
+    assert python.healthy(prefix, C.DEFAULT) is False
+
+
+def test_unhealthy_system_version_changes(python_dir):
+    prefix, tmpdir = python_dir
+
+    python.install_environment(prefix, C.DEFAULT, ())
+
+    with open(prefix.path('py_env-default/pyvenv.cfg'), 'a') as f:
+        f.write('base-executable = /does/not/exist\n')
+
+    assert python.healthy(prefix, C.DEFAULT) is False
+
+
+def test_unhealthy_old_virtualenv(python_dir):
+    prefix, tmpdir = python_dir
+
+    python.install_environment(prefix, C.DEFAULT, ())
+
+    # simulate "old" virtualenv by deleting this file
+    os.remove(prefix.path('py_env-default/pyvenv.cfg'))
 
-        assert python.healthy(prefix, C.DEFAULT) is False
+    assert python.healthy(prefix, C.DEFAULT) is False
diff --git a/tests/repository_test.py b/tests/repository_test.py
--- a/tests/repository_test.py
+++ b/tests/repository_test.py
@@ -33,7 +33,6 @@
 from testing.util import get_resource_path
 from testing.util import skipif_cant_run_docker
 from testing.util import skipif_cant_run_swift
-from testing.util import xfailif_no_venv
 from testing.util import xfailif_windows_no_ruby
 
 
@@ -163,7 +162,6 @@ def test_python_hook_weird_setup_cfg(in_git_dir, tempdir_factory, store):
     )
 
 
-@xfailif_no_venv
 def test_python_venv(tempdir_factory, store):  # pragma: no cover (no venv)
     _test_hook_repo(
         tempdir_factory, store, 'python_venv_hooks_repo',