Repository: pex-tool/pex

Issue:
Pex should trust any host passed via `--index` or `--find-links`.
The underlying pip calls don't assume this and error out, e.g.:
```
E   	WARNING: The repository located at www.antlr3.org is not a trusted or secure host and is being ignored. If this repository is available via HTTPS we recommend you use HTTPS instead, otherwise you may silence this warning and allow it anyway with '--trusted-host www.antlr3.org'.
E   	ERROR: Could not find a version that satisfies the requirement antlr_python_runtime==3.1.3 (from versions: none)
E   	ERROR: No matching distribution found for antlr_python_runtime==3.1.3
...
E   	Exception caught: (pex.resolver.Unsatisfiable)
...
E   	  File "/b/f/w/pants/backend/python/subsystems/pex_build_util.py", line 278, in _resolve_multi
E   	    allow_prereleases=python_setup.resolver_allow_prereleases)
E   	  File "/b/f/w/pex_root/install/pex-2.0.2-py2.py3-none-any.whl.b89b057c31d08e6bebdb4531720b75237c56588e/pex-2.0.2-py2.py3-none-any.whl/pex/resolver.py", line 195, in resolve
E   	    raise Unsatisfiable(str(e))
E   	
E   	Exception message: Executing /usr/bin/python2.7 -s -m pip --disable-pip-version-check --isolated -q --cache-dir /home/nobody/.cache/pants/python_cache/requirements/CPython-2.7.12 download --dest /tmp/tmpj2dt894x/resolved --index-url https://pypi.org/simple/ --find-links http://www.antlr3.org/download/Python/ antlr_python_runtime==3.1.3 failed with 1
```


Discussion:
Related to #803 in that `--trusted-host` is a pip network option, but I can't see a reason to expose this directly vs assume it from `indexes` / `find_links`.

Resolving patch:
diff --git a/pex/pip.py b/pex/pip.py
--- a/pex/pip.py
+++ b/pex/pip.py
@@ -8,6 +8,7 @@
 from collections import deque
 
 from pex import third_party
+from pex.compatibility import urlparse
 from pex.interpreter import PythonInterpreter
 from pex.platforms import Platform
 from pex.variables import ENV
@@ -46,6 +47,16 @@ def execute_pip_isolated(args, cache=None, interpreter=None):
 
 
 def _calculate_package_index_options(indexes=None, find_links=None):
+  trusted_hosts = []
+
+  def maybe_trust_insecure_host(url):
+    url_info = urlparse.urlparse(url)
+    if 'http' == url_info.scheme:
+      # Implicitly trust explicitly asked for http indexes and find_links repos instead of requiring
+      # seperate trust configuration.
+      trusted_hosts.append(url_info.netloc)
+    return url
+
   # N.B.: We interpret None to mean accept pip index defaults, [] to mean turn off all index use.
   if indexes is not None:
     if len(indexes) == 0:
@@ -53,16 +64,20 @@ def _calculate_package_index_options(indexes=None, find_links=None):
     else:
       all_indexes = deque(indexes)
       yield '--index-url'
-      yield all_indexes.popleft()
+      yield maybe_trust_insecure_host(all_indexes.popleft())
       if all_indexes:
         for extra_index in all_indexes:
           yield '--extra-index-url'
-          yield extra_index
+          yield maybe_trust_insecure_host(extra_index)
 
   if find_links:
     for find_link_url in find_links:
       yield '--find-links'
-      yield find_link_url
+      yield maybe_trust_insecure_host(find_link_url)
+
+  for trusted_host in trusted_hosts:
+    yield '--trusted-host'
+    yield trusted_host
 
 
 def download_distributions(target,


Test patch:
diff --git a/tests/test_integration.py b/tests/test_integration.py
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -19,7 +19,6 @@
 from pex.pex_info import PexInfo
 from pex.pip import build_wheels, download_distributions
 from pex.testing import (
-    IS_PYPY,
     NOT_CPYTHON27,
     NOT_CPYTHON27_OR_OSX,
     NOT_CPYTHON36_OR_LINUX,
@@ -1096,9 +1095,7 @@ def test_pex_interpreter_interact_custom_setuptools_useable():
     assert rc == 0, stdout
 
 
-@pytest.mark.skipif(IS_PYPY,
-                    reason='Our pyenv interpreter setup fails under pypy: '
-                           'https://github.com/pantsbuild/pex/issues/477')
+@skip_for_pyenv_use_under_pypy
 def test_setup_python():
   interpreter = ensure_python_interpreter(PY27)
   with temporary_dir() as out:
@@ -1111,9 +1108,7 @@ def test_setup_python():
     subprocess.check_call([pex, '-c', 'import jsonschema'])
 
 
-@pytest.mark.skipif(IS_PYPY,
-                    reason='Our pyenv interpreter setup fails under pypy: '
-                           'https://github.com/pantsbuild/pex/issues/477')
+@skip_for_pyenv_use_under_pypy
 def test_setup_interpreter_constraint():
   interpreter = ensure_python_interpreter(PY27)
   with temporary_dir() as out:
@@ -1133,9 +1128,7 @@ def test_setup_interpreter_constraint():
     assert rc == 0
 
 
-@pytest.mark.skipif(IS_PYPY,
-                    reason='Our pyenv interpreter setup fails under pypy: '
-                           'https://github.com/pantsbuild/pex/issues/477')
+@skip_for_pyenv_use_under_pypy
 def test_setup_python_multiple_transitive_markers():
   py27_interpreter = ensure_python_interpreter(PY27)
   py36_interpreter = ensure_python_interpreter(PY36)
@@ -1169,9 +1162,7 @@ def test_setup_python_multiple_transitive_markers():
     assert to_bytes(os.path.realpath(py36_interpreter)) == stdout.strip()
 
 
-@pytest.mark.skipif(IS_PYPY,
-                    reason='Our pyenv interpreter setup fails under pypy: '
-                           'https://github.com/pantsbuild/pex/issues/477')
+@skip_for_pyenv_use_under_pypy
 def test_setup_python_direct_markers():
   py36_interpreter = ensure_python_interpreter(PY36)
   with temporary_dir() as out:
@@ -1189,9 +1180,7 @@ def test_setup_python_direct_markers():
       subprocess.check_call(py2_only_program, env=make_env(PATH=os.path.dirname(py36_interpreter)))
 
 
-@pytest.mark.skipif(IS_PYPY,
-                    reason='Our pyenv interpreter setup fails under pypy: '
-                           'https://github.com/pantsbuild/pex/issues/477')
+@skip_for_pyenv_use_under_pypy
 def test_setup_python_multiple_direct_markers():
   py36_interpreter = ensure_python_interpreter(PY36)
   py27_interpreter = ensure_python_interpreter(PY27)
@@ -1562,9 +1551,7 @@ def test_pex_reexec_constraints_dont_match_current_pex_python():
                      interpreter_constraints=['=={}'.format(version)])
 
 
-@pytest.mark.skipif(IS_PYPY,
-                    reason='Our pyenv interpreter setup fails under pypy: '
-                           'https://github.com/pantsbuild/pex/issues/477')
+@skip_for_pyenv_use_under_pypy
 def test_issues_745_extras_isolation():
   # Here we ensure one of our extras, `subprocess32`, is properly isolated in the transition from
   # pex bootstrapping where it is imported by `pex.executor` to execution of user code.
@@ -1604,3 +1591,20 @@ def test_issues_745_extras_isolation():
 
     subprocess32_location = os.path.realpath(output.decode('utf-8').strip())
     assert subprocess32_location.startswith(pex_root)
+
+
+@skip_for_pyenv_use_under_pypy
+def test_trusted_host_handling():
+  python = ensure_python_interpreter(PY27)
+  # Since we explicitly ask Pex to find links at http://www.antlr3.org/download/Python, it should
+  # implicitly trust the www.antlr3.org host.
+  results = run_pex_command(
+    args=[
+      '--find-links=http://www.antlr3.org/download/Python',
+      'antlr_python_runtime==3.1.3',
+      '--',
+      '-c', 'import antlr3'
+    ],
+    python=python
+  )
+  results.assert_success()