Repository: pex-tool/pex

Issue:
Teardown of tmp dirs can fail and lead to process failure.
This was seen in CI:
```
=================================== FAILURES ===================================
_____________________ test_reproducible_build_python_flag ______________________
[gw1] linux2 -- Python 2.7.18 /home/runner/work/pex/pex/.tox/py27-integration/bin/python

    def test_reproducible_build_python_flag():
        # type: () -> None
>       assert_reproducible_build(["--python=python2.7"])

tests/test_integration.py:1869: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_integration.py:1799: in assert_reproducible_build
    pexes = create_multiple_pexes()
tests/test_integration.py:1789: in create_multiple_pexes
    create_pex(path, seed=(index * 497) + 4)
tests/test_integration.py:1776: in create_pex
    result.assert_success()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <pex.testing.IntegResults object at 0x7fe7727b0950>

    def assert_success(self):
        # type: () -> None
        assert (
            self.return_code == 0
        ), "integration test failed: return_code={}, output={}, error={}".format(
>           self.return_code, self.output, self.error
        )
E       AssertionError: integration test failed: return_code=-6, output=, error=pex: Building pex
E       pex: Building pex :: Resolving interpreters
E       pex: Building pex :: Resolving distributions ([])
E       pex: Building pex :: Resolving distributions ([]) :: Resolving requirements.
E       pex: Building pex: 13.0ms
E       pex:   Resolving interpreters: 12.2ms
E       pex:   Resolving distributions ([]): 0.2ms
E       pex:     Resolving requirements.: 0.2ms
E       pex: Vendoring attr from VendorSpec(key='attrs', requirement='attrs==20.3.0', rewrite=True, constrain=True) @ /home/runner/work/pex/pex/pex/vendor/_vendored/attrs
E       pex: Vendoring packaging from VendorSpec(key='packaging', requirement='packaging==20.8', rewrite=True, constrain=True) @ /home/runner/work/pex/pex/pex/vendor/_vendored/packaging
E       pex: Vendoring pyparsing from VendorSpec(key='packaging', requirement='packaging==20.8', rewrite=True, constrain=True) @ /home/runner/work/pex/pex/pex/vendor/_vendored/packaging
E       pex: Vendoring pkg_resources from VendorSpec(key='setuptools', requirement='setuptools==44.0.0', rewrite=True, constrain=True) @ /home/runner/work/pex/pex/pex/vendor/_vendored/setuptools
E       pex: Hashing pex
E       Saving PEX file to /tmp/tmp4U4eo7/1.pex
E       pex: Zipping PEX file.
E       pex: Hashing pex: 64.1ms
E       pex: Isolating pex
E       pex: Isolating pex :: Extracting pex to /tmp/tmpln2u1H/isolated/8a88ba3b3f741fdc979a686f8e743b4c12b13bba
E       pex: Zipping PEX file.: 93.4ms
E       Error in atexit._run_exitfuncs:
E       Traceback (most recent call last):
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/atexit.py", line 24, in _run_exitfuncs
E           func(*targs, **kargs)
E         File "pex/common.py", line 160, in teardown
E           self._rmtree(td)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 279, in rmtree
E           onerror(os.rmdir, path, sys.exc_info())
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 277, in rmtree
E           os.rmdir(path)
E       OSError: [Errno 39] Directory not empty: '/tmp/tmpln2u1H/isolated/8a88ba3b3f741fdc979a686f8e743b4c12b13bba.6d1838e2035e4dff85a2ff5579e50555/pex/vendor/_vendored/pip/pip/_vendor'
E       Error in sys.exitfunc:
E       Traceback (most recent call last):
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/atexit.py", line 24, in _run_exitfuncs
E           func(*targs, **kargs)
E         File "pex/common.py", line 160, in teardown
E           self._rmtree(td)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 270, in rmtree
E           rmtree(fullname, ignore_errors, onerror)
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 279, in rmtree
E           onerror(os.rmdir, path, sys.exc_info())
E         File "/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/shutil.py", line 277, in rmtree
E           os.rmdir(path)
E       OSError: [Errno 39] Directory not empty: '/tmp/tmpln2u1H/isolated/8a88ba3b3f741fdc979a686f8e743b4c12b13bba.6d1838e2035e4dff85a2ff5579e50555/pex/vendor/_vendored/pip/pip/_vendor'
E       Fatal Python error: PyImport_GetModuleDict: no module dictionary!

pex/testing.py:342: AssertionError
```

Pex should probably just log warnings at most here:
https://github.com/pantsbuild/pex/blob/26593c5163199d37e6360a0d594f1d713cbcdf1f/pex/common.py#L156-L160




Discussion:

Resolving patch:
diff --git a/pex/common.py b/pex/common.py
--- a/pex/common.py
+++ b/pex/common.py
@@ -139,7 +139,6 @@ def __init__(self):
         self._registry = defaultdict(set)  # type: DefaultDict[int, Set[str]]
         self._lock = threading.RLock()
         self._getpid = os.getpid
-        self._exists = os.path.exists
         self._rmtree = shutil.rmtree
         atexit.register(self.teardown)
 
@@ -156,8 +155,7 @@ def register(self, path):
     def teardown(self):
         # type: () -> None
         for td in self._registry.pop(self._getpid(), []):
-            if self._exists(td):
-                self._rmtree(td)
+            self._rmtree(td, ignore_errors=True)
 
 
 _MKDTEMP_SINGLETON = MktempTeardownRegistry()


Test patch: