Repository: pantsbuild/pants

Issue:
--binaries-path-by-id failing with TypeError when passed on the command line
I'm pretty sure this is a regression (that I probably introduced the last time I modified `binary_util.py`) that needs fixing immediately.

```bash
> ./pants clean-all --async --binaries-path-by-id="{('darwin','17'):('mac','10.12')}"
Exception caught: (<type 'exceptions.TypeError'>)
  File "/Users/dmcclanahan/.cache/pants/pex/bin/pants.pex/1.8.0rc0-a66a8062/pants.pex/.bootstrap/_pex/pex.py", line 367, in execute
    self._wrap_coverage(self._wrap_profiling, self._execute)
  File "/Users/dmcclanahan/.cache/pants/pex/bin/pants.pex/1.8.0rc0-a66a8062/pants.pex/.bootstrap/_pex/pex.py", line 293, in _wrap_coverage
    runner(*args)
  File "/Users/dmcclanahan/.cache/pants/pex/bin/pants.pex/1.8.0rc0-a66a8062/pants.pex/.bootstrap/_pex/pex.py", line 325, in _wrap_profiling
    runner(*args)
  File "/Users/dmcclanahan/.cache/pants/pex/bin/pants.pex/1.8.0rc0-a66a8062/pants.pex/.bootstrap/_pex/pex.py", line 410, in _execute
    return self.execute_entry(self._pex_info.entry_point)
  File "/Users/dmcclanahan/.cache/pants/pex/bin/pants.pex/1.8.0rc0-a66a8062/pants.pex/.bootstrap/_pex/pex.py", line 468, in execute_entry
    return runner(entry_point)
  File "/Users/dmcclanahan/.cache/pants/pex/bin/pants.pex/1.8.0rc0-a66a8062/pants.pex/.bootstrap/_pex/pex.py", line 486, in execute_pkg_resources
    return runner()
  File "/Users/dmcclanahan/workspace/source/.pex/install/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl.b88f8c5708514b0697e611262e3b729ad2195fb7/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl/pants/bin/pants_loader.py", line 69, in main
    PantsLoader.run()
  File "/Users/dmcclanahan/workspace/source/.pex/install/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl.b88f8c5708514b0697e611262e3b729ad2195fb7/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl/pants/bin/pants_loader.py", line 65, in run
    cls.load_and_execute(entrypoint)
  File "/Users/dmcclanahan/workspace/source/.pex/install/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl.b88f8c5708514b0697e611262e3b729ad2195fb7/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl/pants/bin/pants_loader.py", line 58, in load_and_execute
    entrypoint_main()
  File "/Users/dmcclanahan/workspace/source/.pex/install/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl.b88f8c5708514b0697e611262e3b729ad2195fb7/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl/pants/bin/pants_exe.py", line 39, in main
    PantsRunner(exiter, start_time=start_time).run()
  File "/Users/dmcclanahan/workspace/source/.pex/install/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl.b88f8c5708514b0697e611262e3b729ad2195fb7/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl/pants/bin/pants_runner.py", line 39, in run
    return RemotePantsRunner(self._exiter, self._args, self._env, bootstrap_options).run()
  File "/Users/dmcclanahan/workspace/source/.pex/install/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl.b88f8c5708514b0697e611262e3b729ad2195fb7/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl/pants/bin/remote_pants_runner.py", line 161, in run
    port = self._maybe_launch_pantsd()
  File "/Users/dmcclanahan/workspace/source/.pex/install/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl.b88f8c5708514b0697e611262e3b729ad2195fb7/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl/pants/bin/remote_pants_runner.py", line 157, in _maybe_launch_pantsd
    return PantsDaemon.Factory.maybe_launch(bootstrap_options=self._bootstrap_options)
  File "/Users/dmcclanahan/workspace/source/.pex/install/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl.b88f8c5708514b0697e611262e3b729ad2195fb7/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl/pants/pantsd/pants_daemon.py", line 89, in maybe_launch
    if stub_pantsd.needs_restart(stub_pantsd.options_fingerprint):
  File "/Users/dmcclanahan/workspace/source/.pex/install/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl.b88f8c5708514b0697e611262e3b729ad2195fb7/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl/pants/pantsd/pants_daemon.py", line 252, in options_fingerprint
    invert=True
  File "/Users/dmcclanahan/workspace/source/.pex/install/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl.b88f8c5708514b0697e611262e3b729ad2195fb7/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl/pants/option/options_fingerprinter.py", line 59, in combined_options_fingerprint_for_scope
    fingerprinter.fingerprint(option_type, option_value)
  File "/Users/dmcclanahan/workspace/source/.pex/install/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl.b88f8c5708514b0697e611262e3b729ad2195fb7/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl/pants/option/options_fingerprinter.py", line 91, in fingerprint
    return self._fingerprint_primitives(option_val)
  File "/Users/dmcclanahan/workspace/source/.pex/install/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl.b88f8c5708514b0697e611262e3b729ad2195fb7/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl/pants/option/options_fingerprinter.py", line 159, in _fingerprint_primitives
    return stable_json_sha1(val)
  File "/Users/dmcclanahan/workspace/source/.pex/install/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl.b88f8c5708514b0697e611262e3b729ad2195fb7/pantsbuild.pants-1.8.0rc0+a66a8062-cp27-none-macosx_10_11_x86_64.whl/pants/base/hash_utils.py", line 49, in stable_json_hash
    json_str = json.dumps(obj, ensure_ascii=True, allow_nan=False, sort_keys=True, cls=encoder)
  File "/opt/ee/python/2.7/lib/python2.7/json/__init__.py", line 251, in dumps
    sort_keys=sort_keys, **kw).encode(obj)
  File "/opt/ee/python/2.7/lib/python2.7/json/encoder.py", line 209, in encode
    chunks = list(chunks)
  File "/opt/ee/python/2.7/lib/python2.7/json/encoder.py", line 431, in _iterencode
    for chunk in _iterencode_list(o, _current_indent_level):
  File "/opt/ee/python/2.7/lib/python2.7/json/encoder.py", line 332, in _iterencode_list
    for chunk in chunks:
  File "/opt/ee/python/2.7/lib/python2.7/json/encoder.py", line 382, in _iterencode_dict
    raise TypeError("key " + repr(key) + " is not a string")

Exception message: key (u'darwin', u'17') is not a string
```


Discussion:
cli arg parsing issue. putting it under pants.ini works:
```
[tw-mbp-yic pants (nsaechao-nsaechao/issue-6332_install_goal)]$ git diff
diff --git a/pants.ini b/pants.ini
index d0857fb..5a321ea 100644
--- a/pants.ini
+++ b/pants.ini
@@ -22,6 +22,8 @@ local_artifact_cache: %(pants_bootstrapdir)s/artifact_cache
 [GLOBAL]
 print_exception_stacktrace: True
 
+binaries_path_by_id: {('darwin','17'):('mac','10.12')}
+
 # Enable our own custom loose-source plugins as well as contribs.
 pythonpath: [
     "%(buildroot)s/contrib/avro/src/python",


$ ./pants options | grep binaries
binaries_baseurls = ['https://binaries.pantsbuild.org'] (from HARDCODED)
binaries_fetch_timeout_secs = 30 (from HARDCODED)
binaries_path_by_id = {(u'darwin', u'17'): (u'mac', u'10.12')} (from CONFIG in pants.ini)
```
This does not repro on master, I'm seeing if it reproes on 1.8.x and/or 1.9.x, or if it's somehow just our internal pants.
It does not repro on 1.8.x, so closing this as this seems to only occur internally. As @wisechengyi noted, anyone else seeing this issue can probably modify the entry in their pants.ini to work successfully. Please feel free to reopen this issue if it reproes elsewhere, but my suspicion as of now is that it's some feature of our internal python (because that's where the failure occurs).
Definitely repros - needs pantsd. This is current master HEAD:
```
$ ./pants --enable-pantsd --binaries-path-by-id="{('darwin','17'):('mac','10.12')}" clean-all --async
Exception caught: (<type 'exceptions.TypeError'>)
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/bin/pants_loader.py", line 75, in <module>
    main()
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/bin/pants_loader.py", line 71, in main
    PantsLoader.run()
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/bin/pants_loader.py", line 67, in run
    cls.load_and_execute(entrypoint)
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/bin/pants_loader.py", line 60, in load_and_execute
    entrypoint_main()
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/bin/pants_exe.py", line 38, in main
    PantsRunner(exiter, start_time=start_time).run()
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/bin/pants_runner.py", line 39, in run
    return RemotePantsRunner(self._exiter, self._args, self._env, bootstrap_options).run()
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/bin/remote_pants_runner.py", line 161, in run
    port = self._maybe_launch_pantsd()
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/bin/remote_pants_runner.py", line 157, in _maybe_launch_pantsd
    return PantsDaemon.Factory.maybe_launch(bootstrap_options=self._bootstrap_options)
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/pantsd/pants_daemon.py", line 88, in maybe_launch
    if stub_pantsd.needs_restart(stub_pantsd.options_fingerprint):
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/pantsd/pants_daemon.py", line 249, in options_fingerprint
    invert=True
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/option/options_fingerprinter.py", line 53, in combined_options_fingerprint_for_scope
    fingerprint = fingerprinter.fingerprint(option_type, option_value)
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/option/options_fingerprinter.py", line 89, in fingerprint
    return self._fingerprint_primitives(option_val)
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/option/options_fingerprinter.py", line 157, in _fingerprint_primitives
    return stable_json_sha1(val)
  File "/home/jsirois/dev/pantsbuild/pants/src/python/pants/base/hash_utils.py", line 54, in stable_json_hash
    json_str = json.dumps(obj, ensure_ascii=True, allow_nan=False, sort_keys=True, cls=encoder)
  File "/usr/lib64/python2.7/json/__init__.py", line 251, in dumps
    sort_keys=sort_keys, **kw).encode(obj)
  File "/usr/lib64/python2.7/json/encoder.py", line 209, in encode
    chunks = list(chunks)
  File "/usr/lib64/python2.7/json/encoder.py", line 431, in _iterencode
    for chunk in _iterencode_list(o, _current_indent_level):
  File "/usr/lib64/python2.7/json/encoder.py", line 332, in _iterencode_list
    for chunk in chunks:
  File "/usr/lib64/python2.7/json/encoder.py", line 382, in _iterencode_dict
    raise TypeError("key " + repr(key) + " is not a string")

Exception message: key (u'darwin', u'17') is not a string
```
It probably makes sense to add an encoding for tuples -> lists (sets -> lists could use this treatment too) here: https://github.com/pantsbuild/pants/blob/aad3a3caf0a01c7daeda5822b3f2661de7475379/src/python/pants/option/options_fingerprinter.py#L19-L23
Related #4695

Resolving patch:
diff --git a/src/python/pants/option/options_fingerprinter.py b/src/python/pants/option/options_fingerprinter.py
--- a/src/python/pants/option/options_fingerprinter.py
+++ b/src/python/pants/option/options_fingerprinter.py
@@ -10,6 +10,8 @@
 from builtins import object, open, str
 from hashlib import sha1
 
+from future.moves import collections
+
 from pants.base.build_environment import get_buildroot
 from pants.base.hash_utils import stable_json_hash
 from pants.option.custom_types import (UnsetBool, dict_with_files_option, dir_option, file_option,
@@ -20,9 +22,14 @@ class Encoder(json.JSONEncoder):
   def default(self, o):
     if o is UnsetBool:
       return '_UNSET_BOOL_ENCODING'
+    elif isinstance(o, collections.Iterable) and not isinstance(o, collections.Mapping):
+      # For things like sets which JSONEncoder doesn't handle, and raises a TypeError on.
+      return list(o)
     return super(Encoder, self).default(o)
 
 
+# TODO: this function could be wrapped in @memoized -- we would have to manage the cache size
+# somehow if this is done or it might blow up.
 stable_json_sha1 = functools.partial(stable_json_hash, encoder=Encoder)
 
 
@@ -72,6 +79,11 @@ def fingerprint(self, option_type, option_val):
     if option_val is None:
       return None
 
+    # The python documentation (https://docs.python.org/2/library/json.html#json.dumps) states that
+    # dict keys are coerced to strings in json.dumps, but this appears to be incorrect.
+    if isinstance(option_val, collections.Mapping):
+      option_val = {stable_json_sha1(k):v for k, v in option_val.items()}
+
     # For simplicity, we always fingerprint a list.  For non-list-valued options,
     # this will be a singleton list.
     if not isinstance(option_val, (list, tuple)):
@@ -163,6 +175,10 @@ def _fingerprint_dict_with_files(self, option_val):
     contents rather than by its path.
 
     This assumes the files are small enough to be read into memory.
+
+    NB: The keys of the dict are assumed to be strings -- if they are not, the dict should be
+    converted to encode its keys with `stable_json_sha1()`, as is done in the `fingerprint()`
+    method.
     """
     # Dicts are wrapped in singleton lists. See the "For simplicity..." comment in `fingerprint()`.
     option_val = option_val[0]


Test patch:
diff --git a/tests/python/pants_test/option/test_options_fingerprinter.py b/tests/python/pants_test/option/test_options_fingerprinter.py
--- a/tests/python/pants_test/option/test_options_fingerprinter.py
+++ b/tests/python/pants_test/option/test_options_fingerprinter.py
@@ -5,17 +5,43 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import os
+import unittest
 from builtins import range, zip
 
+from future.utils import PY2
+
 from pants.base.payload import Payload
 from pants.base.payload_field import PrimitiveField
 from pants.option.custom_types import (UnsetBool, dict_option, dir_option, file_option, list_option,
                                        target_option)
-from pants.option.options_fingerprinter import OptionsFingerprinter
+from pants.option.options_fingerprinter import OptionsFingerprinter, stable_json_sha1
 from pants.util.contextutil import temporary_dir
 from pants_test.test_base import TestBase
 
 
+class JsonEncodingTest(unittest.TestCase):
+  def test_known_checksums(self):
+    """Check a laundry list of supported inputs to stable_json_sha1().
+
+    This checks both that the method can successfully handle the type of input object, but also that
+    the hash of specific objects remains stable.
+    """
+    self.assertEqual(stable_json_sha1({}), 'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f')
+    self.assertEqual(stable_json_sha1(()), '97d170e1550eee4afc0af065b78cda302a97674c')
+    self.assertEqual(stable_json_sha1([]), '97d170e1550eee4afc0af065b78cda302a97674c')
+    self.assertEqual(stable_json_sha1(set([])), '97d170e1550eee4afc0af065b78cda302a97674c')
+    self.assertEqual(stable_json_sha1([{}]), '4e9950a1f2305f56d358cad23f28203fb3aacbef')
+    self.assertEqual(stable_json_sha1([('a', 3)]), 'd6abed2e53c1595fb3075ecbe020365a47af1f6f')
+    self.assertEqual(stable_json_sha1({'a': 3}), '9e0e6d8a99c72daf40337183358cbef91bba7311')
+    self.assertEqual(stable_json_sha1([{'a': 3}]), '8f4e36849a0b8fbe9c4a822c80fbee047c65458a')
+    self.assertEqual(stable_json_sha1(set([1])), 'f629ae44b7b3dcfed444d363e626edf411ec69a8')
+
+  def test_non_string_dict_key(self):
+    error_regex_str = r'key \(\) is not a string' if PY2 else r'keys must be a string'
+    with self.assertRaisesRegexp(TypeError, error_regex_str):
+      stable_json_sha1({():()})
+
+
 class OptionsFingerprinterTest(TestBase):
 
   def setUp(self):
@@ -31,6 +57,11 @@ def test_fingerprint_dict(self):
     self.assertEqual(fp1, fp2)
     self.assertNotEqual(fp1, fp3)
 
+  def test_fingerprint_dict_with_non_string_keys(self):
+    d = {('a', 2): (3, 4)}
+    fp = self.options_fingerprinter.fingerprint(dict_option, d)
+    self.assertEqual(fp, 'ba3b9319e9477f14dcea56ae1836ae5e73a73a2c')
+
   def test_fingerprint_list(self):
     l1 = [1, 2, 3]
     l2 = [1, 3, 2]