Repository: pantsbuild/pants

Issue:
BinaryUtilIntegrationTest doesn't do what it looks like it does
The call to `build-support/bin/download_binary.sh cmake 3.9.5` is broken - it should be `build-support/bin/download_binary.sh cmake 3.9.5 cmake.tar.gz`. 

The only reason this test passes is that the underlying logic short-circuits the actual fetching if the destination file already exists, and we happen to only run this test in such circumstances. Which means that the test isn't ever actually downloading anything...  

So we need to either turn off the short-circuiting in the fetcher, or ensure that the destination file doesn't exist, before shelling out to the script. However this is actually a little tricky, since there's no easy way to plumb that option to the fetcher when running it via a script, and there's no easy way to know the destination file location in the test. In any case, this test mutates real pants repo state. It's probably best to modify it to run in a hermetic environment, which would also solve the above problem.

Also, it's unclear that the very thin `download_binary.sh` script merits the cost of an integration test. It might be better to call into the underlying code programmatically. That would also allow us to set the destination to a clean one. And if the purpose of the script is to emit a path, then the test should test that the correct path is actually emitted...


Discussion:
See https://github.com/pantsbuild/pants/pull/6681

Resolving patch:
diff --git a/src/python/pants/binaries/binary_util.py b/src/python/pants/binaries/binary_util.py
--- a/src/python/pants/binaries/binary_util.py
+++ b/src/python/pants/binaries/binary_util.py
@@ -487,11 +487,11 @@ def _create_bootstrap_binary_arg_parser():
   return parser
 
 
-def main():
+def select(argv):
   # Parse positional arguments to the script.
-  args = _create_bootstrap_binary_arg_parser().parse_args()
+  args = _create_bootstrap_binary_arg_parser().parse_args(argv[1:])
   # Resolve bootstrap options with a fake empty command line.
-  options_bootstrapper = OptionsBootstrapper.create(args=[sys.argv[0]])
+  options_bootstrapper = OptionsBootstrapper.create(args=[argv[0]])
   subsystems = (GlobalOptionsRegistrar, BinaryUtil.Factory)
   known_scope_infos = reduce(set.union, (ss.known_scope_infos() for ss in subsystems), set())
   options = options_bootstrapper.get_full_options(known_scope_infos)
@@ -521,10 +521,9 @@ def main():
     platform_dependent=True,
     external_url_generator=None,
     archiver=archiver_for_current_binary)
-  path = binary_util.select(binary_request)
 
-  print(path)
+  return binary_util.select(binary_request)
 
 
 if __name__ == '__main__':
-  main()
+  print(select(sys.argv))


Test patch:
diff --git a/tests/python/pants_test/binaries/test_binary_util.py b/tests/python/pants_test/binaries/test_binary_util.py
--- a/tests/python/pants_test/binaries/test_binary_util.py
+++ b/tests/python/pants_test/binaries/test_binary_util.py
@@ -4,6 +4,7 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import glob
 import logging
 import os
 from builtins import object, open, str
@@ -12,10 +13,11 @@
 from future.utils import PY2
 
 from pants.binaries.binary_util import (BinaryRequest, BinaryToolFetcher, BinaryToolUrlGenerator,
-                                        BinaryUtil)
+                                        BinaryUtil, select)
 from pants.net.http.fetcher import Fetcher
-from pants.util.contextutil import temporary_dir
-from pants.util.dirutil import safe_open
+from pants.util.collections import assert_single_element
+from pants.util.contextutil import environment_as, temporary_dir
+from pants.util.dirutil import is_readable_dir, safe_file_dump, safe_open
 from pants_test.test_base import TestBase
 
 
@@ -339,3 +341,21 @@ def test_disallowing_external_urls(self):
       "external_url_generator=ExternalUrlGenerator(<example __str__()>), archiver=None): "
       "--binaries-baseurls is empty.")
     self.assertIn(expected_msg, the_raised_exception_message)
+
+  def test_select_argv(self):
+    """Test invoking binary_util.py as a standalone script."""
+    with temporary_dir() as tmp_dir:
+      config_file_loc = os.path.join(tmp_dir, 'pants.ini')
+      safe_file_dump(config_file_loc, """\
+[GLOBAL]
+allow_external_binary_tool_downloads: True
+pants_bootstrapdir: {}
+""".format(tmp_dir), binary_mode=False)
+      expected_output_glob = os.path.join(
+        tmp_dir, 'bin', 'cmake', '*', '*', '3.9.5', 'cmake')
+      with environment_as(PANTS_CONFIG_FILES='[{!r}]'.format(config_file_loc)):
+        # Ignore the first argument, as per sys.argv.
+        output_file = select(['_', 'cmake', '3.9.5', 'cmake.tar.gz'])
+      self.assertTrue(is_readable_dir(output_file))
+      realized_glob = assert_single_element(glob.glob(expected_output_glob))
+      self.assertEqual(os.path.realpath(output_file), os.path.realpath(realized_glob))
diff --git a/tests/python/pants_test/binaries/test_download_binary_bootstrap_script_integration.py b/tests/python/pants_test/binaries/test_download_binary_bootstrap_script_integration.py
deleted file mode 100644
--- a/tests/python/pants_test/binaries/test_download_binary_bootstrap_script_integration.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# coding=utf-8
-# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
-# Licensed under the Apache License, Version 2.0 (see LICENSE).
-
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-import logging
-import os
-import unittest
-from builtins import str
-
-from pants.base.build_environment import get_buildroot
-from pants.util.process_handler import subprocess
-
-
-logger = logging.getLogger(__name__)
-
-
-class BinaryUtilIntegrationTest(unittest.TestCase):
-
-  script_path = os.path.join(
-    get_buildroot(),
-    'build-support/bin/download_binary.sh',
-  )
-
-  def test_fetch_cmake_success(self):
-    self.assertEqual(
-      0,
-      subprocess.check_call([self.script_path, 'cmake', '3.9.5']))
-
-  def test_fetch_nonexistent_tool_failure(self):
-    try:
-      subprocess.check_output([self.script_path, 'cmake', '213521351235'],
-                              stderr=subprocess.STDOUT)
-      self.fail('Version 213521351235 of cmake should not exist!')
-    except subprocess.CalledProcessError as e:
-      self.assertNotEqual(0, e.returncode)
-      self.assertIn('Failed to fetch cmake binary from any source', str(e.output))