Repository: pantsbuild/pants

Issue:
v2 python pipeline can fail with IOError attempting to open non-existent PEX-INFO file
there seems to be some kind of issue with the v2 python pipeline where chroot dirs may be created without actually containing the appropriate pex guts. subsequent attempts to open the dir as a pex fail with the following traceback:

```
Exception caught: (<type 'exceptions.IOError'>)
  File ".bootstrap/_pex/pex.py", line 360, in execute
    self._wrap_coverage(self._wrap_profiling, self._execute)
  File ".bootstrap/_pex/pex.py", line 288, in _wrap_coverage
    runner(*args)
  File ".bootstrap/_pex/pex.py", line 320, in _wrap_profiling
    runner(*args)
  File ".bootstrap/_pex/pex.py", line 403, in _execute
    return self.execute_entry(self._pex_info.entry_point)
  File ".bootstrap/_pex/pex.py", line 461, in execute_entry
    return runner(entry_point)
  File ".bootstrap/_pex/pex.py", line 479, in execute_pkg_resources
    return runner()
  File "/Users/kwilson/dev/source3/.pex/install/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl.e1e47cc7c0ed42f9079dc57415d84d0475bb8579/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl/pants/bin/pants_exe.py", line 44, in main
    PantsRunner(exiter).run()
  File "/Users/kwilson/dev/source3/.pex/install/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl.e1e47cc7c0ed42f9079dc57415d84d0475bb8579/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl/pants/bin/pants_runner.py", line 57, in run
    options_bootstrapper=options_bootstrapper)
  File "/Users/kwilson/dev/source3/.pex/install/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl.e1e47cc7c0ed42f9079dc57415d84d0475bb8579/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl/pants/bin/pants_runner.py", line 46, in _run
    return LocalPantsRunner(exiter, args, env, options_bootstrapper=options_bootstrapper).run()
  File "/Users/kwilson/dev/source3/.pex/install/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl.e1e47cc7c0ed42f9079dc57415d84d0475bb8579/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl/pants/bin/local_pants_runner.py", line 37, in run
    self._run()
  File "/Users/kwilson/dev/source3/.pex/install/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl.e1e47cc7c0ed42f9079dc57415d84d0475bb8579/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl/pants/bin/local_pants_runner.py", line 79, in _run
    goal_runner_result = goal_runner.run()
  File "/Users/kwilson/dev/source3/.pex/install/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl.e1e47cc7c0ed42f9079dc57415d84d0475bb8579/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl/pants/bin/goal_runner.py", line 263, in run
    result = self._execute_engine()
  File "/Users/kwilson/dev/source3/.pex/install/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl.e1e47cc7c0ed42f9079dc57415d84d0475bb8579/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl/pants/bin/goal_runner.py", line 252, in _execute_engine
    result = engine.execute(self._context, self._goals)
  File "/Users/kwilson/dev/source3/.pex/install/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl.e1e47cc7c0ed42f9079dc57415d84d0475bb8579/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl/pants/engine/legacy_engine.py", line 26, in execute
    self.attempt(context, goals)
  File "/Users/kwilson/dev/source3/.pex/install/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl.e1e47cc7c0ed42f9079dc57415d84d0475bb8579/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl/pants/engine/round_engine.py", line 224, in attempt
    goal_executor.attempt(explain)
  File "/Users/kwilson/dev/source3/.pex/install/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl.e1e47cc7c0ed42f9079dc57415d84d0475bb8579/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl/pants/engine/round_engine.py", line 47, in attempt
    task.execute()
  File "/Users/kwilson/dev/source3/.pex/install/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl.e1e47cc7c0ed42f9079dc57415d84d0475bb8579/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl/pants/backend/python/tasks2/resolve_requirements.py", line 22, in execute
    pex = self.resolve_requirements(req_libs)
  File "/Users/kwilson/dev/source3/.pex/install/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl.e1e47cc7c0ed42f9079dc57415d84d0475bb8579/pantsbuild.pants-1.3.0rc2+4151880633-py2-none-any.whl/pants/backend/python/tasks2/resolve_requirements_task_base.py", line 51, in resolve_requirements
    return PEX(path, interpreter=interpreter)
  File "/Users/kwilson/dev/source3/.pex/install/pex-1.2.6-py2.py3-none-any.whl.cd87d3cea16ee5296bcdaff2f0885852ffe356c9/pex-1.2.6-py2.py3-none-any.whl/pex/pex.py", line 59, in __init__
    self._pex_info = PexInfo.from_pex(self._pex)
  File "/Users/kwilson/dev/source3/.pex/install/pex-1.2.6-py2.py3-none-any.whl.cd87d3cea16ee5296bcdaff2f0885852ffe356c9/pex-1.2.6-py2.py3-none-any.whl/pex/pex_info.py", line 79, in from_pex
    with open(os.path.join(pex, cls.PATH)) as fp:

Exception message: [Errno 2] No such file or directory: u'/Users/kwilson/dev/source3/.pants.d/pyprep/requirements/CPython-2.7.10/6ad744b7087e2b25f09595d0527bdedb3b92c69a/PEX-INFO'
```

where the referenced dir is empty:
```
$ ls -al /Users/kwilson/dev/source3/.pants.d/pyprep/requirements/CPython-2.7.10/6ad744b7087e2b25f09595d0527bdedb3b92c69a
total 0
drwxr-xr-x  2 kwilson  staff   68 Jun  5 18:17 .
drwxr-xr-x  6 kwilson  staff  204 Jun  5 18:17 ..
```


Discussion:
How frequent is this? Worth blocking `1.3.0` over?
fairly frequent (~8 times in the past week, exacerbated by resolve failures afaict). I think the fix should be rather simple. when is `1.3.0` slated?
I think that if this doesn't block `1.3.0` final, then we could/should get `1.3.0` out today. Otherwise, new rc, new clock, etc. If it's intermittent and the patch will be small, maybe backporting it for `1.3.1` would be fine?
`1.3.1` sounds good.
I think this is due to an error (or even just ctrl-c) while creating the resolved requirements pex dir. 

Note that we create it with `safe_concurrent_creation`, but that method moves the tmp path to the real path in the `finally` block, so it happens even when there's an error. This is surely not the right behavior.
current theory:

1) in [ResolveRequirementsTaskBase._build_requirements_pex](https://github.com/pantsbuild/pants/blob/master/src/python/pants/backend/python/tasks2/resolve_requirements_task_base.py#L54-L56), we do:

```
    builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
    dump_requirements(builder, interpreter, req_libs, self.context.log)
    builder.freeze()
```

2) in [PEXBuilder.__init__](https://github.com/pantsbuild/pex/blob/master/pex/pex_builder.py#L78), we instantiate a `Chroot` which does an [eager mkdir here](https://github.com/pantsbuild/pex/blob/master/pex/common.py#L217)

3) if the `dump_requirements()` call fails with a resolve failure, that exception handling appears to be deferred somehow (pants' workunit output continues) - so no requirements are dumped to the empty pex dir - but the eagerly created dir is moved into place via the `safe_concurrent_create` contextmgr.

4) subsequent re-runs fail with the `No such file or directory` issue.


here's a repro in the pants' repo:

  ```
  [omerta pants (master)]$ ls /tmp/pex_interpreter_bootstrap
  setuptools-30.0.0.tar.gz wheel-0.29.0.tar.gz
  [omerta pants (master)]$ ./pants --python-repos-indexes=[] --python-repos-repos="['file:///tmp/pex_interpreter_bootstrap/']" binary src/python/pants/bin:pants
  ...
  Exception message: Could not satisfy all requirements for twitter.common.collections<0.4,>=0.3.1:
      twitter.common.collections<0.4,>=0.3.1
  [omerta pants (master)]$ ./pants --python-repos-indexes=[] --python-repos-repos="['file:///tmp/pex_interpreter_bootstrap/']" binary src/python/pants/bin:pants
  ...
  Exception message: [Errno 2] No such file or directory: u'/Users/kwilson/dev/pants/.pants.d/pyprep/requirements/CPython-2.7.10/a812971972fac74249d400e24b9274853a728871/PEX-INFO'
  ```

Part of the solution must be to fix `safe_concurrent_create`, or, if other callsites expect the current behavior, to create another helper function that does the right thing here.

Resolving patch:
diff --git a/src/python/pants/util/dirutil.py b/src/python/pants/util/dirutil.py
--- a/src/python/pants/util/dirutil.py
+++ b/src/python/pants/util/dirutil.py
@@ -250,7 +250,10 @@ def safe_concurrent_creation(target_path):
   tmp_path = '{}.tmp.{}'.format(target_path, uuid.uuid4().hex)
   try:
     yield tmp_path
-  finally:
+  except Exception:
+    rm_rf(tmp_path)
+    raise
+  else:
     if os.path.exists(tmp_path):
       safe_concurrent_rename(tmp_path, target_path)
 


Test patch:
diff --git a/tests/python/pants_test/util/test_dirutil.py b/tests/python/pants_test/util/test_dirutil.py
--- a/tests/python/pants_test/util/test_dirutil.py
+++ b/tests/python/pants_test/util/test_dirutil.py
@@ -264,7 +264,7 @@ def test_safe_concurrent_creation_noop(self):
       self.assertFalse(os.path.exists(expected_file))
       self.assertTrue(os.path.exists(os.path.dirname(expected_file)))
 
-  def test_safe_concurrent_creation_exception_still_renames(self):
+  def test_safe_concurrent_creation_exception_handling(self):
     with temporary_dir() as td:
       expected_file = os.path.join(td, 'expected_file')
 
@@ -275,7 +275,7 @@ def test_safe_concurrent_creation_exception_still_renames(self):
           raise ZeroDivisionError('zomg')
 
       self.assertFalse(os.path.exists(safe_path))
-      self.assertTrue(os.path.exists(expected_file))
+      self.assertFalse(os.path.exists(expected_file))
 
   def test_safe_rm_oldest_items_in_dir(self):
     with temporary_dir() as td: