Repository: pypa/virtualenv

Issue:
virtualenv fails sometimes when run concurrently
**Issue**

Sometimes our CI builds fail when creating virtualenv. We've tracked the problem to builds running in parallel, trying to create a virtualenv in different directories at the same time. 
Able to reproduce it with a simple test:
```console
rm -rf *; parallel python3.7 -m virtualenv  ::: $(seq 10)
```
**WARNING**: run in a scratch directory (note `rm -rf *`).

Most of the times the test passes, but sometimes it fails with errors like these:
```console
$ parallel python3.7 -m virtualenv --clear ::: $(seq 10)
created virtual environment CPython3.7.7.final.0-64 in 169ms
  creator CPython3Posix(dest=/home/jenkins/virtualenv/2, clear=True, global=False)
  seeder FromAppData(download=False, pip=latest, setuptools=latest, wheel=latest, via=copy, app_data_dir=/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1)
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator
Exception in thread Thread-5:
Traceback (most recent call last):
  File "/opt/starfish/python3.7-shared/lib/python3.7/threading.py", line 926, in _bootstrap_inner
    self.run()
  File "/opt/starfish/python3.7-shared/lib/python3.7/threading.py", line 870, in run
    self._target(*self._args, **self._kwargs)
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/via_app_data/via_app_data.py", line 50, in _install
    installer.build_image()
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/via_app_data/pip_install/base.py", line 59, in build_image
    zip_ref.extractall(str(self._image_dir))
  File "/opt/starfish/python3.7-shared/lib/python3.7/zipfile.py", line 1636, in extractall
    self._extract_member(zipinfo, path, pwd)
  File "/opt/starfish/python3.7-shared/lib/python3.7/zipfile.py", line 1690, in _extract_member
    open(targetpath, "wb") as target:
FileNotFoundError: [Errno 2] No such file or directory: '/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1/3.7/image/CopyPipInstall/wheel-0.34.2-py2.py3-none-any/wheel-0.34.2.dist-info/RECORD'

Exception in thread Thread-6:
Traceback (most recent call last):
  File "/opt/starfish/python3.7-shared/lib/python3.7/threading.py", line 926, in _bootstrap_inner
    self.run()
  File "/opt/starfish/python3.7-shared/lib/python3.7/threading.py", line 870, in run
    self._target(*self._args, **self._kwargs)
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/via_app_data/via_app_data.py", line 50, in _install
    installer.build_image()
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/via_app_data/pip_install/base.py", line 59, in build_image
    zip_ref.extractall(str(self._image_dir))
  File "/opt/starfish/python3.7-shared/lib/python3.7/zipfile.py", line 1636, in extractall
    self._extract_member(zipinfo, path, pwd)
  File "/opt/starfish/python3.7-shared/lib/python3.7/zipfile.py", line 1682, in _extract_member
    os.makedirs(upperdirs)
  File "/opt/starfish/python3.7-shared/lib/python3.7/os.py", line 223, in makedirs
    mkdir(name, mode)
FileNotFoundError: [Errno 2] No such file or directory: '/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1/3.7/image/CopyPipInstall/pip-20.0.2-py2.py3-none-any/pip/_internal/cli'

Exception in thread Thread-4:
Traceback (most recent call last):
  File "/opt/starfish/python3.7-shared/lib/python3.7/threading.py", line 926, in _bootstrap_inner
    self.run()
  File "/opt/starfish/python3.7-shared/lib/python3.7/threading.py", line 870, in run
    self._target(*self._args, **self._kwargs)
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/via_app_data/via_app_data.py", line 50, in _install
    installer.build_image()
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/via_app_data/pip_install/base.py", line 59, in build_image
    zip_ref.extractall(str(self._image_dir))
  File "/opt/starfish/python3.7-shared/lib/python3.7/zipfile.py", line 1636, in extractall
    self._extract_member(zipinfo, path, pwd)
  File "/opt/starfish/python3.7-shared/lib/python3.7/zipfile.py", line 1682, in _extract_member
    os.makedirs(upperdirs)
  File "/opt/starfish/python3.7-shared/lib/python3.7/os.py", line 213, in makedirs
    makedirs(head, exist_ok=exist_ok)
  File "/opt/starfish/python3.7-shared/lib/python3.7/os.py", line 223, in makedirs
    mkdir(name, mode)
FileNotFoundError: [Errno 2] No such file or directory: '/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1/3.7/image/CopyPipInstall/setuptools-46.1.3-py3-none-any/pkg_resources/_vendor'

created virtual environment CPython3.7.7.final.0-64 in 151ms
  creator CPython3Posix(dest=/home/jenkins/virtualenv/1, clear=True, global=False)
  seeder FromAppData(download=False, pip=latest, setuptools=latest, wheel=latest, via=copy, app_data_dir=/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1)
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator
Exception in thread Thread-2:
Traceback (most recent call last):
  File "/opt/starfish/python3.7-shared/lib/python3.7/threading.py", line 926, in _bootstrap_inner
    self.run()
  File "/opt/starfish/python3.7-shared/lib/python3.7/threading.py", line 870, in run
    self._target(*self._args, **self._kwargs)
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/via_app_data/via_app_data.py", line 75, in _get
    self.app_data,
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/embed/wheels/acquire.py", line 28, in get_wheels
    acquire_from_bundle(processed, for_py_version, wheel_cache_dir)
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/embed/wheels/acquire.py", line 57, in acquire_from_bundle
    copy2(str(bundle), str(bundled_wheel_file))
  File "/opt/starfish/python3.7-shared/lib/python3.7/shutil.py", line 267, in copy2
    copystat(src, dst, follow_symlinks=follow_symlinks)
  File "/opt/starfish/python3.7-shared/lib/python3.7/shutil.py", line 206, in copystat
    follow_symlinks=follow)
FileNotFoundError: [Errno 2] No such file or directory

Exception in thread Thread-1:
Traceback (most recent call last):
  File "/opt/starfish/python3.7-shared/lib/python3.7/threading.py", line 926, in _bootstrap_inner
    self.run()
  File "/opt/starfish/python3.7-shared/lib/python3.7/threading.py", line 870, in run
    self._target(*self._args, **self._kwargs)
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/via_app_data/via_app_data.py", line 75, in _get
    self.app_data,
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/embed/wheels/acquire.py", line 28, in get_wheels
    acquire_from_bundle(processed, for_py_version, wheel_cache_dir)
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/embed/wheels/acquire.py", line 57, in acquire_from_bundle
    copy2(str(bundle), str(bundled_wheel_file))
  File "/opt/starfish/python3.7-shared/lib/python3.7/shutil.py", line 267, in copy2
    copystat(src, dst, follow_symlinks=follow_symlinks)
  File "/opt/starfish/python3.7-shared/lib/python3.7/shutil.py", line 206, in copystat
    follow_symlinks=follow)
FileNotFoundError: [Errno 2] No such file or directory

Exception in thread Thread-4:
Traceback (most recent call last):
  File "/opt/starfish/python3.7-shared/lib/python3.7/threading.py", line 926, in _bootstrap_inner
    self.run()
  File "/opt/starfish/python3.7-shared/lib/python3.7/threading.py", line 870, in run
    self._target(*self._args, **self._kwargs)
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/via_app_data/via_app_data.py", line 50, in _install
    installer.build_image()
  File "/opt/starfish/python3.7-shared/lib/python3.7/site-packages/virtualenv/seed/via_app_data/pip_install/base.py", line 58, in build_image
    with zipfile.ZipFile(str(self._wheel)) as zip_ref:
  File "/opt/starfish/python3.7-shared/lib/python3.7/zipfile.py", line 1240, in __init__
    self.fp = io.open(file, filemode)
FileNotFoundError: [Errno 2] No such file or directory: '/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1/3.7/wheels/wheel-0.34.2-py2.py3-none-any.whl'

created virtual environment CPython3.7.7.final.0-64 in 293ms
  creator CPython3Posix(dest=/home/jenkins/virtualenv/8, clear=True, global=False)
  seeder FromAppData(download=False, pip=latest, setuptools=latest, wheel=latest, via=copy, app_data_dir=/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1)
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator
created virtual environment CPython3.7.7.final.0-64 in 349ms
  creator CPython3Posix(dest=/home/jenkins/virtualenv/10, clear=True, global=False)
  seeder FromAppData(download=False, pip=latest, setuptools=latest, wheel=latest, via=copy, app_data_dir=/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1)
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator
created virtual environment CPython3.7.7.final.0-64 in 402ms
  creator CPython3Posix(dest=/home/jenkins/virtualenv/5, clear=True, global=False)
  seeder FromAppData(download=False, pip=latest, setuptools=latest, wheel=latest, via=copy, app_data_dir=/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1)
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator
created virtual environment CPython3.7.7.final.0-64 in 462ms
  creator CPython3Posix(dest=/home/jenkins/virtualenv/4, clear=True, global=False)
  seeder FromAppData(download=False, pip=latest, setuptools=latest, wheel=latest, via=copy, app_data_dir=/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1)
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator
created virtual environment CPython3.7.7.final.0-64 in 509ms
  creator CPython3Posix(dest=/home/jenkins/virtualenv/9, clear=True, global=False)
  seeder FromAppData(download=False, pip=latest, setuptools=latest, wheel=latest, via=copy, app_data_dir=/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1)
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator
created virtual environment CPython3.7.7.final.0-64 in 572ms
  creator CPython3Posix(dest=/home/jenkins/virtualenv/3, clear=True, global=False)
  seeder FromAppData(download=False, pip=latest, setuptools=latest, wheel=latest, via=copy, app_data_dir=/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1)
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator
created virtual environment CPython3.7.7.final.0-64 in 625ms
  creator CPython3Posix(dest=/home/jenkins/virtualenv/7, clear=True, global=False)
  seeder FromAppData(download=False, pip=latest, setuptools=latest, wheel=latest, via=copy, app_data_dir=/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1)
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator
created virtual environment CPython3.7.7.final.0-64 in 714ms
  creator CPython3Posix(dest=/home/jenkins/virtualenv/6, clear=True, global=False)
  seeder FromAppData(download=False, pip=latest, setuptools=latest, wheel=latest, via=copy, app_data_dir=/home/jenkins/.local/share/virtualenv/seed-app-data/v1.0.1)
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator
```

**Environment**

Provide at least:
- OS: Ubuntu 16, Ubuntu 18, CentOS 7
- ``pip list`` of the host python where ``virtualenv`` is installed:
```console
$ python3.7 -m pip freeze --all
appdirs==1.4.3
distlib==0.3.0
filelock==3.0.12
importlib-metadata==1.6.0
pip==20.0.2
setuptools==46.1.3
six==1.14.0
virtualenv==20.0.18
wheel==0.34.2
zipp==3.1.0
```

**Output of the virtual environment creation**

See above


Discussion:
This feels like is file lock bug, in particular, the use case described by https://github.com/benediktschmitt/py-filelock/issues/27#issuecomment-536073252

PS. This is not the case.
Ok, so got to the bottom of this. Seems we're deleting the application data cache folder when --clear is passed, due to the following subtlety:

```python
from argparse import ArgumentParser

parser = ArgumentParser()
parser.add_argument("--clear", action="store_true", default=False)
parser.add_argument("--clear-app-data", action="store_true", default=False)

opts = parser.parse_args(["--clear"])
print(opts)

# prints Namespace(clear=True, clear_app_data=False)
``` 

however:

```python
from argparse import ArgumentParser

parser = ArgumentParser()
parser.add_argument("--clear-app-data", action="store_true", default=False)

opts = parser.parse_args(["--clear"])
print(opts)

# prints Namespace(clear_app_data=True)
``` 

😲😲😲😲😲🤦‍♂️🤦‍♂️🤦‍♂️🤦‍♂️🤦‍♂️

Resolving patch:
diff --git a/src/virtualenv/run/__init__.py b/src/virtualenv/run/__init__.py
--- a/src/virtualenv/run/__init__.py
+++ b/src/virtualenv/run/__init__.py
@@ -58,8 +58,8 @@ def build_parser(args=None, options=None):
         help="a data folder used as cache by the virtualenv",
     )
     parser.add_argument(
-        "--clear-app-data",
-        dest="clear_app_data",
+        "--reset-app-data",
+        dest="reset_app_data",
         action="store_true",
         help="start with empty app data folder",
         default=False,
diff --git a/src/virtualenv/run/plugin/discovery.py b/src/virtualenv/run/plugin/discovery.py
--- a/src/virtualenv/run/plugin/discovery.py
+++ b/src/virtualenv/run/plugin/discovery.py
@@ -24,7 +24,7 @@ def get_discover(parser, args):
     options, _ = parser.parse_known_args(args)
     if options.app_data == "<temp folder>":
         options.app_data = TempAppData()
-    if options.clear_app_data:
+    if options.reset_app_data:
         options.app_data.clean()
     discover_class = discover_types[options.discovery]
     discover_class.add_parser_arguments(discovery_parser)


Test patch:
diff --git a/tests/unit/config/cli/test_parser.py b/tests/unit/config/cli/test_parser.py
--- a/tests/unit/config/cli/test_parser.py
+++ b/tests/unit/config/cli/test_parser.py
@@ -5,8 +5,9 @@
 
 import pytest
 
-from virtualenv.config.cli.parser import VirtualEnvConfigParser
+from virtualenv.config.cli.parser import VirtualEnvConfigParser, VirtualEnvOptions
 from virtualenv.config.ini import IniConfig
+from virtualenv.run import session_via_cli
 
 
 @pytest.fixture()
@@ -36,3 +37,10 @@ def test_flag(gen_parser_no_conf_env):
     assert result.clear is False
     result = run("--clear")
     assert result.clear is True
+
+
+def test_reset_app_data_does_not_conflict_clear():
+    options = VirtualEnvOptions()
+    session_via_cli(["--clear", "venv"], options=options)
+    assert options.clear is True
+    assert options.reset_app_data is False
diff --git a/tests/unit/seed/test_boostrap_link_via_app_data.py b/tests/unit/seed/test_boostrap_link_via_app_data.py
--- a/tests/unit/seed/test_boostrap_link_via_app_data.py
+++ b/tests/unit/seed/test_boostrap_link_via_app_data.py
@@ -31,7 +31,7 @@ def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies)
         bundle_ver["pip"].split("-")[1],
         "--setuptools",
         bundle_ver["setuptools"].split("-")[1],
-        "--clear-app-data",
+        "--reset-app-data",
         "--creator",
         current_fastest,
         "-vv",
@@ -119,7 +119,7 @@ def read_only_folder(temp_app_data):
 @pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files")
 def test_base_bootstrap_link_via_app_data_not_writable(tmp_path, current_fastest, read_only_folder, monkeypatch):
     dest = tmp_path / "venv"
-    result = cli_run(["--seeder", "app-data", "--creator", current_fastest, "--clear-app-data", "-vv", str(dest)])
+    result = cli_run(["--seeder", "app-data", "--creator", current_fastest, "--reset-app-data", "-vv", str(dest)])
     assert result
 
 
diff --git a/tests/unit/seed/test_extra_install.py b/tests/unit/seed/test_extra_install.py
--- a/tests/unit/seed/test_extra_install.py
+++ b/tests/unit/seed/test_extra_install.py
@@ -39,7 +39,7 @@ def builtin_shows_marker_missing():
 def test_can_build_c_extensions(creator, tmp_path, coverage_env):
     env, greet = tmp_path / "env", str(tmp_path / "greet")
     shutil.copytree(str(Path(__file__).parent.resolve() / "greet"), greet)
-    session = cli_run(["--creator", creator, "--seed", "app-data", str(env), "-vvv"])
+    session = cli_run(["--creator", creator, "--seeder", "app-data", str(env), "-vvv"])
     coverage_env()
     cmd = [
         str(session.creator.script("pip")),