Repository: pantsbuild/pants

Issue:
Allow for deprecation of child scopes
### Problem

Currently an attempt to deprecate a scope below another scope (eg: deprecating the `test.mytask` scope in favor of `mytask`) will fail 50% of the time based on registration order due to the requirement that scopes are registered in alphabetical order, and governed by a requirement that parent scopes are frozen before children.

These requirements mean that if the new scope for a deprecation sorts before the old scope, the deprecated sub scope will already have options registered in it by the time the parent is visited, triggering a `FrozenRegistration` error.

### Solution

This form of deprecation is very useful (the "`test`" example above is real-ish, and triggered by welcome v2 changes in the `test` scope). The parser freezing case appeared to be in service of an optimization, but removing it and replacing it with eager lookups to possibly shadowed options 1) is clearer, 2) has equivalent performance.

### Result

Nested option scopes may be deprecated. Fixes #8984.


Discussion:

Resolving patch:
diff --git a/src/python/pants/option/errors.py b/src/python/pants/option/errors.py
--- a/src/python/pants/option/errors.py
+++ b/src/python/pants/option/errors.py
@@ -28,10 +28,6 @@ class BooleanOptionNameWithNo(RegistrationError):
   """"Boolean option names cannot start with --no."""
 
 
-class FrozenRegistration(RegistrationError):
-  """Cannot register an option on a scope after registering on any of its inner scopes."""
-
-
 class ImplicitValIsNone(RegistrationError):
   """Implicit value cannot be None."""
 
diff --git a/src/python/pants/option/parser.py b/src/python/pants/option/parser.py
--- a/src/python/pants/option/parser.py
+++ b/src/python/pants/option/parser.py
@@ -44,7 +44,6 @@
   BooleanConversionError,
   BooleanOptionNameWithNo,
   FromfileError,
-  FrozenRegistration,
   ImplicitValIsNone,
   InvalidKwarg,
   InvalidKwargNonGlobalScope,
@@ -76,10 +75,8 @@ class Parser:
   a child of the 'compile' scope, which is a child of the global scope.
 
   Options registered on a parser are also registered transitively on all the scopes it encloses.
-  Registration must be in outside-in order: we forbid registering options on an outer scope if
-  we've already registered an option on one of its inner scopes. This is to ensure that
-  re-registering the same option name on an inner scope correctly replaces the identically-named
-  option from the outer scope.
+  We forbid registering options that shadow other options, and registration walks up and down the
+  hierarchy to enforce that.
   """
 
   @staticmethod
@@ -102,6 +99,15 @@ def _invert(cls, s: Optional[Union[bool, str]]) -> Optional[bool]:
     b = cls._ensure_bool(s)
     return not b
 
+  @classmethod
+  def scope_str(cls, scope: str) -> str:
+    return 'global scope' if scope == GLOBAL_SCOPE else f"scope '{scope}'"
+
+  @classmethod
+  def _check_shadowing(cls, parent_scope, parent_known_args, child_scope, child_known_args):
+    for arg in (parent_known_args & child_known_args):
+      raise Shadowing(child_scope, arg, outer_scope=cls.scope_str(parent_scope))
+
   def __init__(
     self,
     env: Dict[str, str],
@@ -125,9 +131,6 @@ def __init__(
     self._scope = self._scope_info.scope
     self._option_tracker = option_tracker
 
-    # If True, no more registration is allowed on this parser.
-    self._frozen = False
-
     # All option args registered with this parser.  Used to prevent shadowing args in inner scopes.
     self._known_args: Set[str] = set()
 
@@ -434,18 +437,9 @@ def _recursive_option_registration_args(self):
 
   def register(self, *args, **kwargs) -> None:
     """Register an option."""
-    if self._frozen:
-      raise FrozenRegistration(self.scope, args[0])
-
     if args:
       name, dest = self.parse_name_and_dest(*args, **kwargs)
       self._check_deprecated(name, kwargs, print_warning=False)
-    
-    # Prevent further registration in enclosing scopes.
-    ancestor = self._parent_parser
-    while ancestor:
-      ancestor._freeze()
-      ancestor = ancestor._parent_parser
 
     if kwargs.get('type') == bool:
       default = kwargs.get('default')
@@ -459,11 +453,15 @@ def register(self, *args, **kwargs) -> None:
 
     # Record the args. We'll do the underlying parsing on-demand.
     self._option_registrations.append((args, kwargs))
-    if self._parent_parser:
-      for arg in args:
-        existing_scope = self._parent_parser._existing_scope(arg)
-        if existing_scope is not None:
-          raise Shadowing(self.scope, arg, outer_scope=self._scope_str(existing_scope))
+
+    # Look for shadowing options up and down the hierarchy.
+    args_set = set(args)
+    for parent in self._parents_transitive():
+      self._check_shadowing(parent.scope, parent._known_args, self.scope, args_set)
+    for child in self._children_transitive():
+      self._check_shadowing(self.scope, args_set, child.scope, child._known_args)
+
+    # And look for direct conflicts
     for arg in args:
       if arg in self._known_args:
         raise OptionAlreadyRegistered(self.scope, arg)
@@ -537,13 +535,16 @@ def error(
     if removal_version is not None:
       validate_deprecation_semver(removal_version, 'removal version')
 
-  def _existing_scope(self, arg: str) -> Optional[str]:
-    if arg in self._known_args:
-      return self._scope
-    elif self._parent_parser:
-      return self._parent_parser._existing_scope(arg)
-    else:
-      return None
+  def _parents_transitive(self):
+    ancestor = self._parent_parser
+    while ancestor:
+      yield ancestor
+      ancestor = ancestor._parent_parser
+
+  def _children_transitive(self):
+    for child in self._child_parsers:
+      yield child
+      yield from child._children_transitive()
 
   _ENV_SANITIZER_RE = re.compile(r'[.-]')
 
@@ -767,12 +768,8 @@ def _inverse_arg(self, arg: str) -> Optional[str]:
   def _register_child_parser(self, child: "Parser") -> None:
     self._child_parsers.append(child)
 
-  def _freeze(self) -> None:
-    self._frozen = True
-
   def _scope_str(self, scope: Optional[str] = None) -> str:
-    scope = scope or self.scope
-    return 'global scope' if scope == GLOBAL_SCOPE else f"scope '{scope}'"
+    return self.scope_str(scope if scope is not None else self.scope)
 
   def __str__(self) -> str:
     return f'Parser({self._scope})'


Test patch:
diff --git a/src/python/pants/option/options_test.py b/src/python/pants/option/options_test.py
--- a/src/python/pants/option/options_test.py
+++ b/src/python/pants/option/options_test.py
@@ -22,7 +22,6 @@
   BooleanConversionError,
   BooleanOptionNameWithNo,
   FromfileError,
-  FrozenRegistration,
   ImplicitValIsNone,
   InvalidKwarg,
   InvalidMemberType,
@@ -726,13 +725,6 @@ def assertError(expected_error, *args, **kwargs):
     assertError(InvalidMemberType, '--foo', type=list, member_type=list)
     assertError(InvalidMemberType, '--foo', type=list, member_type=list)
 
-  def test_frozen_registration(self) -> None:
-    options = Options.create(args=[], env={}, config=self._create_config(),
-                             known_scope_infos=[task('foo')])
-    options.register('foo', '--arg1')
-    with self.assertRaises(FrozenRegistration):
-      options.register(GLOBAL_SCOPE, '--arg2')
-
   def test_implicit_value(self) -> None:
     def check(*, flag: str = "", expected: str) -> None:
       options = self._parse(flags=flag)
@@ -754,6 +746,7 @@ def assert_raises_shadowing(*, scope: str, args: List[str]) -> None:
       with self.assertRaises(Shadowing):
         options.register(scope, *args)
 
+    assert_raises_shadowing(scope='', args=['--opt2'])
     assert_raises_shadowing(scope='bar', args=['--opt1'])
     assert_raises_shadowing(scope='foo.bar', args=['--opt1'])
     assert_raises_shadowing(scope='foo.bar', args=['--opt2'])
@@ -1524,6 +1517,48 @@ class DummyOptionable2(Optionable):
     # Check values.
     self.assertEqual('uu', vals2.qux)
 
+  def test_scope_deprecation_parent(self) -> None:
+    # Note: This test demonstrates that a scope can mark itself as deprecating a subscope of
+    # another scope.
+    class DummyOptionable1(Optionable):
+      options_scope = 'test'
+      options_scope_category = ScopeInfo.SUBSYSTEM
+
+      @classmethod
+      def register_options(cls, register):
+        super().register_options(register)
+        register('--bar')
+
+    class DummyOptionable2(Optionable):
+      options_scope = 'lint'
+      options_scope_category = ScopeInfo.SUBSYSTEM
+      deprecated_options_scope = 'test.a-bit-linty'
+      deprecated_options_scope_removal_version = '9999.9.9.dev0'
+
+      @classmethod
+      def register_options(cls, register):
+        super().register_options(register)
+        register('--foo')
+
+    known_scope_infos = list(DummyOptionable1.known_scope_infos()) +list(DummyOptionable2.known_scope_infos())
+
+    options = Options.create(env={},
+                             config=self._create_config(),
+                             known_scope_infos=known_scope_infos,
+                             args=shlex.split('./pants --test-a-bit-linty-foo=vv'))
+
+    # NB: Order matters here, because Optionables are typically registered in sorted order.
+    DummyOptionable2.register_options_on_scope(options)
+    DummyOptionable1.register_options_on_scope(options)
+
+    with self.warnings_catcher() as w:
+      vals = options.for_scope(DummyOptionable2.options_scope)
+
+    # Check that we got a warning, but also the correct value.
+    single_warning_dummy1 = assert_single_element(w)
+    self.assertEqual(single_warning_dummy1.category, DeprecationWarning)
+    self.assertEqual('vv', vals.foo)
+
   def test_scope_deprecation_defaults(self) -> None:
     # Confirms that a DEFAULT option does not trigger deprecation warnings for a deprecated scope.
     class DummyOptionable1(Optionable):