Repository: pantsbuild/pants

Issue:
deprecate -q as a recursive option
Running a pants command as `./pants -q <args...>` suppresses all output, but putting `-q` after one of the goals causes pants to display other workunit output except for that goal (e.g. `./pants run -q <args...>`). This could be useful, but it seems far more likely to be confusing that `-q` is accepted as an argument, but doesn't actually quiet all the output, and for the case of `./pants run -q <args...>`, the user almost definitely wants to have only the output of running the executable, e.g. for consumption in a script. It might clarify things a bit to deprecate the recursive behavior.


Discussion:
hm. I think that makes sense.
(this arose from an issue I picked up last week internally, so there is at least one person who has run into this)
+1. Not a fan of the recursive options in general, as they end up installed in a bunch of contexts where they don't make sense. They existed before Subsystems, and now I would recommend that a task that wanted to opt into a recursive option would want to depend on a scoped Subsystem explicitly. 

@benjyw : Any thoughts on how/where to actually go about deprecating this?
Yeah, the only truly useful recursive option is `-l`, so you can control logging on a per-task (or per-subsystem) basis. 
I don't know if we need to deprecate this? Is anyone really relying on the recursive behavior of `-q`?
The reason to deprecate it would be the reason in the description: the option is effectively silently ignored in any but the "global" position.
No, I mean I don't know if we need to go through a deprecation cycle for it. I think we can just kill it outright.
Oh, sorry. I see a few dozen cases of the broken behaviour in our repo, but it's fairly greppable...
```
rg -- './pants .* -q'
```
A (long) deprecation would be preferable I think.

Resolving patch:
diff --git a/src/python/pants/base/deprecated.py b/src/python/pants/base/deprecated.py
--- a/src/python/pants/base/deprecated.py
+++ b/src/python/pants/base/deprecated.py
@@ -5,10 +5,12 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import inspect
+import sys
 import warnings
 from functools import wraps
 
 import six
+from future.utils import PY2
 from packaging.version import InvalidVersion, Version
 
 from pants.util.memo import memoized_method
@@ -81,7 +83,24 @@ def validate_removal_semver(removal_version):
                                  '{}'.format(removal_version, e))
 
 
-def warn_or_error(removal_version, deprecated_entity_description, hint=None, stacklevel=3):
+def get_frame_info(stacklevel, context=1):
+  """Get a Traceback for the given `stacklevel`.
+
+  For example:
+  `stacklevel=0` means this function's frame (get_frame_info()).
+  `stacklevel=1` means the calling function's frame.
+  See https://docs.python.org/2/library/inspect.html#inspect.getouterframes for more info.
+
+  NB: If `stacklevel` is greater than the number of actual frames, the outermost frame is used
+  instead.
+  """
+  frame_list = inspect.getouterframes(inspect.currentframe(), context=context)
+  frame_stack_index = stacklevel if stacklevel < len(frame_list) else len(frame_list) - 1
+  return frame_list[frame_stack_index]
+
+
+def warn_or_error(removal_version, deprecated_entity_description, hint=None, stacklevel=3,
+                  frame_info=None, ensure_stderr=False):
   """Check the removal_version against the current pants version.
 
   Issues a warning if the removal version is > current pants version, or an error otherwise.
@@ -92,7 +111,12 @@ def warn_or_error(removal_version, deprecated_entity_description, hint=None, sta
                                             we can embed in warning/error messages.
   :param string hint: A message describing how to migrate from the removed entity.
   :param int stacklevel: The stacklevel to pass to warnings.warn.
+  :param FrameInfo frame_info: If provided, use this frame info instead of getting one from
+                               `stacklevel`.
+  :param bool ensure_stderr: Whether use warnings.warn, or use warnings.showwarning to print
+                             directly to stderr.
   :raises DeprecationApplicationError: if the removal_version parameter is invalid.
+  :raises CodeRemovedError: if the current version is later than the version marked for removal.
   """
   removal_semver = validate_removal_semver(removal_version)
 
@@ -101,8 +125,30 @@ def warn_or_error(removal_version, deprecated_entity_description, hint=None, sta
   if hint:
     msg += '\n  {}'.format(hint)
 
+  # We need to have filename and line_number for warnings.formatwarning, which appears to be the only
+  # way to get a warning message to display to stderr. We get that from frame_info -- it's too bad
+  # we have to reconstruct the `stacklevel` logic ourselves, but we do also gain the ability to have
+  # multiple lines of context, which is neat.
+  if frame_info is None:
+    frame_info = get_frame_info(stacklevel)
+  _, filename, line_number, _, code_context, _ = frame_info
+  if code_context:
+    context_lines = ''.join(code_context)
+  else:
+    context_lines = '<no code context available>'
+
   if removal_semver > PANTS_SEMVER:
-    warnings.warn(msg, DeprecationWarning, stacklevel=stacklevel)
+    if ensure_stderr:
+      warning_msg = warnings.formatwarning(
+        msg, DeprecationWarning, filename, line_number, line=context_lines)
+      print(warning_msg, file=sys.stderr)
+    else:
+      warnings.showwarning(
+        DeprecationWarning(msg) if PY2 else msg,
+        DeprecationWarning,
+        filename,
+        line_number,
+        line=context_lines)
   else:
     raise CodeRemovedError(msg)
 
@@ -129,7 +175,7 @@ def deprecated_conditional(predicate,
     warn_or_error(removal_version, entity_description, hint_message, stacklevel=stacklevel)
 
 
-def deprecated(removal_version, hint_message=None, subject=None):
+def deprecated(removal_version, hint_message=None, subject=None, ensure_stderr=False):
   """Marks a function or method as deprecated.
 
   A removal version must be supplied and it must be greater than the current 'pantsbuild.pants'
@@ -147,6 +193,7 @@ def deprecated(removal_version, hint_message=None, subject=None):
   :param str hint_message: An optional hint pointing to alternatives to the deprecation.
   :param str subject: The name of the subject that has been deprecated for logging clarity. Defaults
                       to the name of the decorated function/method.
+  :param bool ensure_stderr: Forwarded to `ensure_stderr` in warn_or_error().
   :raises DeprecationApplicationError if the @deprecation is applied improperly.
   """
   validate_removal_semver(removal_version)
@@ -159,7 +206,8 @@ def decorator(func):
 
     @wraps(func)
     def wrapper(*args, **kwargs):
-      warn_or_error(removal_version, subject or func_full_name, hint_message)
+      warn_or_error(removal_version, subject or func_full_name, hint_message,
+                    ensure_stderr=ensure_stderr)
       return func(*args, **kwargs)
     return wrapper
   return decorator
diff --git a/src/python/pants/option/options.py b/src/python/pants/option/options.py
--- a/src/python/pants/option/options.py
+++ b/src/python/pants/option/options.py
@@ -5,16 +5,30 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import copy
+import re
 import sys
 from builtins import object, open, str
+from textwrap import dedent
 
-from pants.base.deprecated import warn_or_error
+from pants.base.deprecated import get_frame_info, warn_or_error
 from pants.option.arg_splitter import GLOBAL_SCOPE, ArgSplitter
 from pants.option.global_options import GlobalOptionsRegistrar
 from pants.option.option_util import is_list_option
 from pants.option.option_value_container import OptionValueContainer
 from pants.option.parser_hierarchy import ParserHierarchy, all_enclosing_scopes, enclosing_scope
 from pants.option.scope import ScopeInfo
+from pants.util.memo import memoized_property
+
+
+def make_flag_regex(long_name, short_name=None):
+  long_esc = re.escape(long_name)
+  if short_name:
+    short_esc = re.escape(short_name)
+    rx_str = r"\A(?:\-({short_esc})|\-\-(?:(no)\-)?({long_esc})(?:=(.*))?)\Z".format(
+      short_esc=short_esc, long_esc=long_esc)
+  else:
+    rx_str = r"\A\-\-(?:(no)\-)?({long_esc})(?:=(.*))?\Z".format(long_esc=long_esc)
+  return re.compile(rx_str)
 
 
 class Options(object):
@@ -285,6 +299,84 @@ def walk_parsers(self, callback):
     self._assert_not_frozen()
     self._parser_hierarchy.walk(callback)
 
+  def _check_deprecated_scope(self, scope, values):
+    # If we're the new name of a deprecated scope, also get values from that scope.
+    deprecated_scope = self.known_scope_to_info[scope].deprecated_scope
+    # Note that deprecated_scope and scope share the same Optionable class, so deprecated_scope's
+    # Optionable has a deprecated_options_scope equal to deprecated_scope. Therefore we must
+    # check that scope != deprecated_scope to prevent infinite recursion.
+    if deprecated_scope is not None and scope != deprecated_scope:
+      # Do the deprecation check only on keys that were explicitly set on the deprecated scope
+      # (and not on its enclosing scopes).
+      explicit_keys = self.for_scope(deprecated_scope,
+                                     inherit_from_enclosing_scope=False).get_explicit_keys()
+      if explicit_keys:
+        # Update our values with those of the deprecated scope (now including values inherited
+        # from its enclosing scope).
+        # Note that a deprecated val will take precedence over a val of equal rank.
+        # This makes the code a bit neater.
+        values.update(self.for_scope(deprecated_scope))
+
+        return dict(
+          removal_version=self.known_scope_to_info[scope].deprecated_scope_removal_version,
+          deprecated_entity_description='scope {}'.format(deprecated_scope),
+          hint='Use scope {} instead (options: {})'.format(scope, ', '.join(explicit_keys)))
+
+  _quiet_flag_regex = make_flag_regex(long_name='quiet', short_name='q')
+
+  # NB: This method can be removed entirely after the deprecation period is complete.
+  def _match_recursive_quiet_flag(self, scope, flags):
+    if scope == GLOBAL_SCOPE:
+      # NB: We do not match a global '-q' or '--quiet' flag -- those are still allowed! Returning
+      # None is explicitly checked for in the _check_deprecations() method.
+      return None
+
+    try:
+      found_flag = next(iter(filter(self._quiet_flag_regex.match, flags)))
+    except StopIteration:
+      # No
+      return None
+
+    if found_flag:
+      return dict(
+        removal_version='1.13.0.dev0',
+        deprecated_entity_description=(
+          "Using the -q or --quiet option recursively "
+          "(i.e. after a goal name on the command line)"),
+        hint=dedent("""
+The -q or --quiet flag should be specified globally by providing it on the
+command line before any other goals (e.g. `./pants -q run
+<binary_target>`). Using -q or --quiet globally will silence all pants logging
+for all tasks and only print output from a console task, which makes binaries
+run from the pants command line usable for consumption by other scripts.
+
+The flag provided was {flag!r}, in scope {scope!r}.
+        """
+        .format(flag=found_flag, scope=scope)),
+        # This shows the call to `self._match_recursive_quiet_flag(...)` in the lambda in
+        # `self.flag_matchers` in the error message so users can know to look at this method for
+        # more context.
+        frame_info=get_frame_info(stacklevel=2),
+        # This warning should always go to stderr, even if a global quiet flag is passed in.
+        ensure_stderr=True)
+
+  @memoized_property
+  def flag_matchers(self):
+    return [
+      lambda scope, _, values: self._check_deprecated_scope(scope, values),
+      lambda scope, flags, _: self._match_recursive_quiet_flag(scope, flags),
+    ]
+
+  def _check_deprecations(self, scope, flags, values):
+    for flag_matcher in self.flag_matchers:
+      maybe_deprecation_warning_kwargs = flag_matcher(scope, flags, values)
+
+      if maybe_deprecation_warning_kwargs is not None:
+        if not isinstance(maybe_deprecation_warning_kwargs, dict):
+          raise TypeError("scope_flags_fun must return a dict, or None (was: {!r})"
+                          .format(maybe_deprecation_warning_kwargs))
+        warn_or_error(**maybe_deprecation_warning_kwargs)
+
   # TODO: Eagerly precompute backing data for this?
   def for_scope(self, scope, inherit_from_enclosing_scope=True):
     """Return the option values for the given scope.
@@ -308,25 +400,8 @@ def for_scope(self, scope, inherit_from_enclosing_scope=True):
     flags_in_scope = self._scope_to_flags.get(scope, [])
     self._parser_hierarchy.get_parser_by_scope(scope).parse_args(flags_in_scope, values)
 
-    # If we're the new name of a deprecated scope, also get values from that scope.
-    deprecated_scope = self.known_scope_to_info[scope].deprecated_scope
-    # Note that deprecated_scope and scope share the same Optionable class, so deprecated_scope's
-    # Optionable has a deprecated_options_scope equal to deprecated_scope. Therefore we must
-    # check that scope != deprecated_scope to prevent infinite recursion.
-    if deprecated_scope is not None and scope != deprecated_scope:
-      # Do the deprecation check only on keys that were explicitly set on the deprecated scope
-      # (and not on its enclosing scopes).
-      explicit_keys = self.for_scope(deprecated_scope,
-                                     inherit_from_enclosing_scope=False).get_explicit_keys()
-      if explicit_keys:
-        warn_or_error(self.known_scope_to_info[scope].deprecated_scope_removal_version,
-                      'scope {}'.format(deprecated_scope),
-                      'Use scope {} instead (options: {})'.format(scope, ', '.join(explicit_keys)))
-        # Update our values with those of the deprecated scope (now including values inherited
-        # from its enclosing scope).
-        # Note that a deprecated val will take precedence over a val of equal rank.
-        # This makes the code a bit neater.
-        values.update(self.for_scope(deprecated_scope))
+    # Check for any deprecation conditions, which are evaluated using `self._flag_matchers`.
+    self._check_deprecations(scope, flags_in_scope, values)
 
     # Cache the values.
     self._values_by_scope[scope] = values


Test patch:
diff --git a/tests/python/pants_test/base/BUILD b/tests/python/pants_test/base/BUILD
--- a/tests/python/pants_test/base/BUILD
+++ b/tests/python/pants_test/base/BUILD
@@ -74,6 +74,7 @@ python_tests(
   dependencies = [
     '3rdparty/python:future',
     'src/python/pants/base:deprecated',
+    'src/python/pants/util:collections',
     'src/python/pants:version',
   ]
 )
diff --git a/tests/python/pants_test/base/test_deprecated.py b/tests/python/pants_test/base/test_deprecated.py
--- a/tests/python/pants_test/base/test_deprecated.py
+++ b/tests/python/pants_test/base/test_deprecated.py
@@ -13,6 +13,7 @@
                                    CodeRemovedError, MissingRemovalVersionError,
                                    NonDevRemovalVersionError, deprecated, deprecated_conditional,
                                    deprecated_module, warn_or_error)
+from pants.util.collections import assert_single_element
 from pants.version import PANTS_SEMVER, VERSION
 
 
@@ -24,9 +25,8 @@ def _test_deprecation(self, deprecation_expected=True):
     with warnings.catch_warnings(record=True) as seen_warnings:
       def assert_deprecation_warning():
         if deprecation_expected:
-          self.assertEqual(1, len(seen_warnings))
-          warning = seen_warnings[0]
-          self.assertIsInstance(warning.message, DeprecationWarning)
+          warning = assert_single_element(seen_warnings)
+          self.assertEqual(warning.category, DeprecationWarning)
           return warning.message
         else:
           self.assertEqual(0, len(seen_warnings))
diff --git a/tests/python/pants_test/option/BUILD b/tests/python/pants_test/option/BUILD
--- a/tests/python/pants_test/option/BUILD
+++ b/tests/python/pants_test/option/BUILD
@@ -17,9 +17,7 @@ python_tests(
 
 python_tests(
   name='options_integration',
-  sources=[
-    'test_options_integration.py',
-  ],
+  sources=globs('*_integration.py'),
   dependencies=[
     '3rdparty/python:future',
     'src/python/pants/util:contextutil',
diff --git a/tests/python/pants_test/option/test_options.py b/tests/python/pants_test/option/test_options.py
--- a/tests/python/pants_test/option/test_options.py
+++ b/tests/python/pants_test/option/test_options.py
@@ -13,6 +13,8 @@
 from contextlib import contextmanager
 from textwrap import dedent
 
+from future.utils import text_type
+
 from pants.base.deprecated import CodeRemovedError
 from pants.engine.fs import FileContent
 from pants.option.arg_splitter import GLOBAL_SCOPE
@@ -31,6 +33,7 @@
 from pants.option.parser import Parser
 from pants.option.ranked_value import RankedValue
 from pants.option.scope import ScopeInfo
+from pants.util.collections import assert_single_element
 from pants.util.contextutil import temporary_file, temporary_file_path
 from pants.util.dirutil import safe_mkdtemp
 
@@ -48,6 +51,31 @@ def subsystem(scope):
 
 
 class OptionsTest(unittest.TestCase):
+
+  @contextmanager
+  def _write_config_to_file(self, fp, config):
+    for section, options in config.items():
+      fp.write('[{}]\n'.format(section))
+      for key, value in options.items():
+        fp.write('{}: {}\n'.format(key, value))
+
+  def _create_config(self, config):
+    with open(os.path.join(safe_mkdtemp(), 'test_config.ini'), 'w') as fp:
+      self._write_config_to_file(fp, config)
+    return Config.load(config_paths=[fp.name])
+
+  def _parse(self, args_str, env=None, config=None, bootstrap_option_values=None, options_cls=None):
+    args = shlex.split(str(args_str))
+    options_type = options_cls or Options
+    options = options_type.create(env=env or {},
+                                  config=self._create_config(config or {}),
+                                  known_scope_infos=OptionsTest._known_scope_infos,
+                                  args=args,
+                                  bootstrap_option_values=bootstrap_option_values,
+                                  option_tracker=OptionTracker())
+    self._register(options)
+    return options
+
   _known_scope_infos = [intermediate('compile'),
                         task('compile.java'),
                         task('compile.scala'),
@@ -159,29 +187,6 @@ def register_global(*args, **kwargs):
     options.register('fingerprinting', '--fingerprinted', fingerprint=True)
     options.register('fingerprinting', '--definitely-not-fingerprinted', fingerprint=False)
 
-  @contextmanager
-  def _write_config_to_file(self, fp, config):
-    for section, options in config.items():
-      fp.write('[{}]\n'.format(section))
-      for key, value in options.items():
-        fp.write('{}: {}\n'.format(key, value))
-
-  def _create_config(self, config):
-    with open(os.path.join(safe_mkdtemp(), 'test_config.ini'), 'w') as fp:
-      self._write_config_to_file(fp, config)
-    return Config.load(config_paths=[fp.name])
-
-  def _parse(self, args_str, env=None, config=None, bootstrap_option_values=None):
-    args = shlex.split(str(args_str))
-    options = Options.create(env=env or {},
-                             config=self._create_config(config or {}),
-                             known_scope_infos=OptionsTest._known_scope_infos,
-                             args=args,
-                             bootstrap_option_values=bootstrap_option_values,
-                             option_tracker=OptionTracker())
-    self._register(options)
-    return options
-
   def test_env_type_int(self):
     options = Options.create(env={'PANTS_FOO_BAR': "['123','456']"},
                              config=self._create_config({}),
@@ -792,6 +797,8 @@ def test_drop_flag_values(self):
 
   def test_deprecated_option_past_removal(self):
     """Ensure that expired options raise CodeRemovedError on attempted use."""
+    # NB: these exceptions are triggered by the `Parser#parse_args()` method, not
+    # `Options#for_scope()`!
     # Test option past removal from flag
     with self.assertRaises(CodeRemovedError):
       self._parse('./pants --global-crufty-expired=way2crufty').for_global_scope()
@@ -811,12 +818,25 @@ def warnings_catcher(self):
       yield w
 
   def assertWarning(self, w, option_string):
-    self.assertEqual(1, len(w))
-    self.assertTrue(issubclass(w[-1].category, DeprecationWarning))
-    warning_message = str(w[-1].message.args)
-    self.assertIn("will be removed in version",
-                  warning_message)
-    self.assertIn(option_string, warning_message)
+    single_warning = assert_single_element(w)
+    self.assertEqual(single_warning.category, DeprecationWarning)
+    warning_message = single_warning.message
+    self.assertIn("will be removed in version", text_type(warning_message))
+    self.assertIn(option_string, text_type(warning_message))
+
+  def test_arbitrary_deprecation_matcher(self):
+    def _matcher(_scope, flags, _values):
+      if any(f.startswith('--int-choices') for f in flags):
+        return dict(removal_version='9999.9.9.dev0',
+                    deprecated_entity_description='int choices test option')
+
+    class OptionsWithDeprecation(Options):
+      flag_matchers = [_matcher]
+
+    with self.warnings_catcher() as w:
+      options = self._parse('./pants --int-choices=42', options_cls=OptionsWithDeprecation)
+      self.assertEqual(options.for_global_scope().int_choices, [42])
+      self.assertWarning(w, 'int choices')
 
   def test_deprecated_options_flag(self):
     with self.warnings_catcher() as w:
@@ -1289,9 +1309,9 @@ class DummyOptionable2(Optionable):
       vals1 = options.for_scope(DummyOptionable1.options_scope)
 
     # Check that we got a warning, but not for the inherited option.
-    self.assertEqual(1, len(w))
-    self.assertTrue(isinstance(w[0].message, DeprecationWarning))
-    self.assertNotIn('inherited', w[0].message.args)
+    single_warning_dummy1 = assert_single_element(w)
+    self.assertEqual(single_warning_dummy1.category, DeprecationWarning)
+    self.assertNotIn('inherited', single_warning_dummy1.message)
 
     # Check values.
     # Deprecated scope takes precedence at equal rank.
@@ -1304,9 +1324,9 @@ class DummyOptionable2(Optionable):
       vals2 = options.for_scope(DummyOptionable2.options_scope)
 
     # Check that we got a warning.
-    self.assertEqual(1, len(w))
-    self.assertTrue(isinstance(w[0].message, DeprecationWarning))
-    self.assertNotIn('inherited', w[0].message.args)
+    single_warning_dummy2 = assert_single_element(w)
+    self.assertEqual(single_warning_dummy2.category, DeprecationWarning)
+    self.assertNotIn('inherited', single_warning_dummy2.message)
 
     # Check values.
     self.assertEqual('uu', vals2.qux)
diff --git a/tests/python/pants_test/option/test_options_integration.py b/tests/python/pants_test/option/test_options_integration.py
--- a/tests/python/pants_test/option/test_options_integration.py
+++ b/tests/python/pants_test/option/test_options_integration.py
@@ -237,6 +237,18 @@ def test_command_line_option_unused_by_goals(self):
     self.assert_success(self.run_pants(['goals', '--bundle-jvm-archive=zip']))
     self.assert_failure(self.run_pants(['goals', '--jvm-invalid=zip']))
 
+  def test_recursive_quiet_deprecated(self):
+    pants_run = self.run_pants(['options', '-q'])
+    self.assert_success(pants_run)
+    self.assertIn("DeprecationWarning: DEPRECATED: Using the -q or --quiet option recursively",
+                  pants_run.stderr_data)
+
+  def test_non_recursive_quiet_no_output(self):
+    pants_run = self.run_pants(['-q', 'compile'])
+    self.assert_success(pants_run)
+    self.assertEqual('', pants_run.stdout_data)
+    self.assertEqual('\n', pants_run.stderr_data)
+
   def test_skip_inherited(self):
     pants_run = self.run_pants([
       '--no-colors', '--no-jvm-platform-validate-colors', '--test-junit-colors',