Repository: pantsbuild/pants

Issue:
pants.ini [binaries] section no longer applies after move to bootstrap options
in #4846 I landed a change to move the binaries options to bootstrap options. this seems to have had the unintended side effect of making the `[binaries]` section no longer work in `pants.ini`:

```
ERROR] Invalid option 'path_by_id' under [binaries] in /Users/kwilson/dev/source/pants.ini
ERROR] Invalid option 'fetch_timeout_secs' under [binaries] in /Users/kwilson/dev/source/pants.ini
ERROR] Invalid option 'baseurls' under [binaries] in /Users/kwilson/dev/source/pants.ini
...

Exception message: Invalid config entries detected. See log for details on which entries to update or remove.
(Specify --no-verify-config to disable this check.)
```

flattening these as e.g.

```
[GLOBAL]
binaries_baseurls: ...
binaries_fetch_timeout_secs: ...
binaries_path_by_id: ...
```

works as expected tho. 

it's not clear to me if this is a violation of the deprecation policy, but I'm also not sure how we can more cleanly migrate these options to bootstrap. posting here mainly for posterity but also to raise a discussion on the broader issue.


Discussion:
One idea might be to add a `deprecated_scope` option to individual registered options. It's already supported for entire scopes/`Optionable`s, so I don't suspect it would be too hard...? https://github.com/pantsbuild/pants/blob/b170aeb3fac52502ff248cf7ceac6172ad6fc75b/src/python/pants/option/options.py#L81
The cmd-line flags (and even env vars) still work, right?

I tried to naively see if `binaries` can be a deprecated scope for the global scope. But no - that leads to shadowing violations. 

We used to have a script that "migrates" the config file. How about resuscitating that, and running it at pants startup during the deprecation cycle? It could error out but give a sensible message on how to fix.

Or - since this is a one-off kind of thing, we could hard-code some special-casing in the config parser...
@benjyw : Is my suggestion of allowing the deprecated scope to be specified on individual options reasonable?
the migration script approach is interesting, but iirc that only provided suggestions and had to be manually run. I'll have to think more about that idea, but it'd be ideal to not have to ship a sidecar solution I think.

I'm currently leaning towards special casing a plural `deprecated_options_scope` on the `GlobalOptionsRegistrar` for cases where scopes have been subsumed by global options. (e.g. `deprecated_options_scopes = ('binaries', 'pantsd', 'watchman')`.

still trying to figure out how best to wire that all up - let me know if you have any high level design thoughts.

Resolving patch:
diff --git a/src/python/pants/option/config.py b/src/python/pants/option/config.py
--- a/src/python/pants/option/config.py
+++ b/src/python/pants/option/config.py
@@ -14,6 +14,9 @@
 from twitter.common.collections import OrderedSet
 
 from pants.base.build_environment import get_buildroot, get_pants_cachedir, get_pants_configdir
+from pants.base.deprecated import deprecated_conditional
+from pants.option.arg_splitter import GLOBAL_SCOPE_CONFIG_SECTION
+from pants.option.global_options import GlobalOptionsRegistrar
 from pants.util.eval import parse_expression
 from pants.util.meta import AbstractClass
 
@@ -53,9 +56,29 @@ def load(cls, configpaths, seed_values=None):
       parser = cls._create_parser(seed_values)
       with open(configpath, 'r') as ini:
         parser.readfp(ini)
+      cls._transform_sections_to_global(parser, GlobalOptionsRegistrar.options_subsumed_scopes)
       single_file_configs.append(_SingleFileConfig(configpath, parser))
     return _ChainedConfig(single_file_configs)
 
+  @staticmethod
+  def _transform_sections_to_global(parser, global_subsumed_sections):
+    """Transforms section names as needed for options scope deprecation."""
+    default_keys = parser.defaults().keys()
+    for subsumed_section, removal_version in global_subsumed_sections:
+      if parser.has_section(subsumed_section):
+        deprecated_conditional(
+          lambda: True,
+          removal_version,
+          'The pants.ini options scope `[{}]` is deprecated. Please migrate options '
+          'in this scope to `[GLOBAL]`.'.format(subsumed_section)
+        )
+        if not parser.has_section(GLOBAL_SCOPE_CONFIG_SECTION):
+          parser.add_section(GLOBAL_SCOPE_CONFIG_SECTION)
+        for k, v in parser.items(subsumed_section):
+          if k not in default_keys:
+            parser.set(GLOBAL_SCOPE_CONFIG_SECTION, '_'.join((subsumed_section, k)), v)
+        parser.remove_section(subsumed_section)
+
   @classmethod
   def _create_parser(cls, seed_values=None):
     """Creates a config parser that supports %([key-name])s value substitution.
diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py
--- a/src/python/pants/option/global_options.py
+++ b/src/python/pants/option/global_options.py
@@ -20,6 +20,11 @@
 class GlobalOptionsRegistrar(Optionable):
   options_scope = GLOBAL_SCOPE
   options_scope_category = ScopeInfo.GLOBAL
+  options_subsumed_scopes = [
+    ('binaries', '1.6.0.dev0'),
+    ('pantsd', '1.6.0.dev0'),
+    ('watchman', '1.6.0.dev0')
+  ]
 
   @classmethod
   def register_bootstrap_options(cls, register):
@@ -174,14 +179,15 @@ def register_bootstrap_options(cls, register):
     # BinaryUtil options.
     register('--binaries-baseurls', type=list, advanced=True,
              default=['https://binaries.pantsbuild.org'],
-             help='List of URLs from which binary tools are downloaded. URLs are searched in '
-                  'order until the requested path is found.')
+             help='List of URLs from which binary tools are downloaded. URLs are '
+                  'searched in order until the requested path is found.')
     register('--binaries-fetch-timeout-secs', type=int, default=30, advanced=True,
              help='Timeout in seconds for URL reads when fetching binary tools from the '
                   'repos specified by --baseurls.')
     register('--binaries-path-by-id', type=dict, advanced=True,
              help=('Maps output of uname for a machine to a binary search path. e.g. '
-             '{("darwin", "15"): ["mac", "10.11"]), ("linux", "arm32"): ["linux", "arm32"]}'))
+                   '{("darwin", "15"): ["mac", "10.11"]), ("linux", "arm32"): ["linux"'
+                   ', "arm32"]}'))
 
     # Pants Daemon options.
     register('--pantsd-pailgun-host', advanced=True, default='127.0.0.1',


Test patch:
diff --git a/tests/python/pants_test/option/test_options_integration.py b/tests/python/pants_test/option/test_options_integration.py
--- a/tests/python/pants_test/option/test_options_integration.py
+++ b/tests/python/pants_test/option/test_options_integration.py
@@ -291,3 +291,34 @@ def test_pants_ignore_option(self):
       self.assertIn("pants_ignore = ['.*/', '/dist/', 'some/random/dir'] (from CONFIG in {})"
                     .format(config_path),
                     pants_run.stdout_data)
+
+  @ensure_engine
+  def test_subsumed_options_scope_config(self):
+    with temporary_dir(root_dir=os.path.abspath('.')) as tempdir:
+      config_path = os.path.relpath(os.path.join(tempdir, 'config.ini'))
+      with open(config_path, 'w+') as f:
+        f.write(dedent("""
+          [binaries]
+          fetch_timeout_secs: 30
+
+          [pantsd]
+          log_dir: /tmp/logs
+
+          [watchman]
+          socket_path: /tmp/test.sock
+        """))
+
+      pants_run = self.run_pants(['--pants-config-files={}'.format(config_path), 'options'])
+      self.assert_success(pants_run)
+
+      template = (
+        'DEPRECATED: The pants.ini options scope `[{}]` is deprecated. Please '
+        'migrate options in this scope to `[GLOBAL]`. will be removed in version'
+      )
+      for scope in ('binaries', 'pantsd', 'watchman'):
+        self.assertIn(template.format(scope), pants_run.stderr_data)
+
+      for option in ('binaries_baseurls', 'pantsd_log_dir', 'watchman_socket_path'):
+        for line in pants_run.stdout_data:
+          if line.startswith(option):
+            self.assertIn('from CONFIG', line)