Repository: pantsbuild/pants

Issue:
exclude-target-regexp not applied consistently in v2
The v2 engine does not apply the `--exclude-target-regexp` option consistently... or, perhaps, it applies it _more_ consistently than v1 does.

As discovered in https://github.com/twitter/commons/pull/451 , the v1 engine apparently applies the excludes to target roots, whereas v2 applies them for all requests for a target.

An example of the failure in v2:
```
$ ./pants --exclude-target-regexp=src/python/twitter/common/rpc/sasl:sourceonly list ./src/python/twitter/common/rpc::
Exception caught: (<class 'pants.build_graph.address_lookup_error.AddressLookupError'>)

Exception message: "sourceonly" was not found in namespace "src/python/twitter/common/rpc/sasl". Did you mean one of:
  :sasl
```
...vs the success in v1:
```
$ ./pants --no-enable-v2-engine --exclude-target-regexp=src/python/twitter/common/rpc/sasl:sourceonly list ./src/python/twitter/common/rpc::
Fetching https://raw.github.com/twitter/finagle/master/finagle-thrift/src/main/thrift/tracing.thrift
src/python/twitter/common/rpc/transports:all
src/python/twitter/common/rpc:rpc-packaged
src/python/twitter/common/rpc:rpc
src/python/twitter/common/rpc/transports:tsslsocket
src/python/twitter/common/rpc/finagle:finagle
src/python/twitter/common/rpc/sasl:sasl
```


Discussion:
I'll tackle this tonight unless someone else thinks they can finish it today.
I saw this behavior before. I was trying to change v1 behavior into v2's. I sent out a review a while ago, but didn't land it.
It seems you are thinking the other way around (changing v2 behavior into v1)?
Yea... It seems like the v1 behavior makes more sense. We have
`--build-ignore` for the "completely ignore a broken file" case, so this
option would seem to be primarily for filtering roots.

On May 11, 2017 10:09 AM, "JieGhost" <notifications@github.com> wrote:

> I saw this behavior before. I was trying to change v1 behavior into v2's.
> I sent out a review a while ago, but didn't land it.
> It seems you are thinking the other way around (changing v2 behavior into
> v1)?
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/pantsbuild/pants/issues/4573#issuecomment-300854860>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AAC2lLuDyHG3gFHq3wwjPDm73l9LTznXks5r40DBgaJpZM4NW5q9>
> .
>

@mateor : you guys use this option, right? And @cheister : Do you?

Resolving patch:
diff --git a/src/python/pants/engine/build_files.py b/src/python/pants/engine/build_files.py
--- a/src/python/pants/engine/build_files.py
+++ b/src/python/pants/engine/build_files.py
@@ -82,8 +82,7 @@ def parse_address_family(address_mapper, path, build_files):
     address_maps.append(AddressMap.parse(filecontent_product.path,
                                          filecontent_product.content,
                                          address_mapper.symbol_table_cls,
-                                         address_mapper.parser_cls,
-                                         address_mapper.exclude_patterns))
+                                         address_mapper.parser_cls))
   return AddressFamily.create(path.path, address_maps)
 
 
@@ -231,9 +230,10 @@ def _hydrate(item_type, spec_path, **kwargs):
 
 
 @rule(BuildFileAddresses,
-      [SelectDependencies(AddressFamily, BuildDirs, field_types=(Dir,)),
+      [Select(AddressMapper),
+       SelectDependencies(AddressFamily, BuildDirs, field_types=(Dir,)),
        Select(_SPECS_CONSTRAINT)])
-def addresses_from_address_families(address_families, spec):
+def addresses_from_address_families(address_mapper, address_families, spec):
   """Given a list of AddressFamilies and a Spec, return matching Addresses.
 
   Raises a ResolveError if:
@@ -243,13 +243,23 @@ def addresses_from_address_families(address_families, spec):
   if not address_families:
     raise ResolveError('Path "{}" contains no BUILD files.'.format(spec.directory))
 
+  def exclude_address(address):
+    if address_mapper.exclude_patterns:
+      address_str = address.spec
+      return any(p.search(address_str) is not None for p in address_mapper.exclude_patterns)
+    return False
+
   if type(spec) in (DescendantAddresses, SiblingAddresses, AscendantAddresses):
-    addresses = tuple(a for af in address_families for a in af.addressables.keys())
+    addresses = tuple(a
+                      for af in address_families
+                      for a in af.addressables.keys()
+                      if not exclude_address(a))
   elif type(spec) is SingleAddress:
     # TODO Could assert len(address_families) == 1, as it should always be true in this case.
     addresses = tuple(a
                       for af in address_families
-                      for a in af.addressables.keys() if a.target_name == spec.name)
+                      for a in af.addressables.keys()
+                      if a.target_name == spec.name and not exclude_address(a))
     if not addresses:
       if len(address_families) == 1:
         _raise_did_you_mean(address_families[0], spec.name)
diff --git a/src/python/pants/engine/mapper.py b/src/python/pants/engine/mapper.py
--- a/src/python/pants/engine/mapper.py
+++ b/src/python/pants/engine/mapper.py
@@ -39,16 +39,8 @@ class AddressMap(datatype('AddressMap', ['path', 'objects_by_name'])):
   :param objects_by_name: A dict mapping from object name to the parsed 'thin' addressable object.
   """
 
-  @staticmethod
-  def exclude_target(target_address, exclude_patterns):
-    if exclude_patterns:
-      for pattern in exclude_patterns:
-        if pattern.search(target_address) is not None:
-          return True
-    return False
-
   @classmethod
-  def parse(cls, filepath, filecontent, symbol_table_cls, parser_cls, exclude_patterns=None):
+  def parse(cls, filepath, filecontent, symbol_table_cls, parser_cls):
     """Parses a source for addressable Serializable objects.
 
     No matter the parser used, the parsed and mapped addressable objects are all 'thin'; ie: any
@@ -61,7 +53,6 @@ def parse(cls, filepath, filecontent, symbol_table_cls, parser_cls, exclude_patt
     :type symbol_table_cls: A :class:`pants.engine.parser.SymbolTable`.
     :param parser_cls: The parser cls to use.
     :type parser_cls: A :class:`pants.engine.parser.Parser`.
-    :param list exclude_patterns: A list of compiled regular expression objects.
     """
     try:
       objects = parser_cls.parse(filepath, filecontent, symbol_table_cls)
@@ -80,10 +71,7 @@ def parse(cls, filepath, filecontent, symbol_table_cls, parser_cls, exclude_patt
       if name in objects_by_name:
         raise DuplicateNameError('An object already exists at {!r} with name {!r}: {!r}.  Cannot '
                                  'map {!r}'.format(filepath, name, objects_by_name[name], obj))
-
-      target_address = '{}:{}'.format(os.path.dirname(filepath), name)
-      if not AddressMap.exclude_target(target_address, exclude_patterns):
-        objects_by_name[name] = obj
+      objects_by_name[name] = obj
     return cls(filepath, OrderedDict(sorted(objects_by_name.items())))
 
 
diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py
--- a/src/python/pants/option/global_options.py
+++ b/src/python/pants/option/global_options.py
@@ -163,8 +163,7 @@ def register_options(cls, register):
                   "are used.  Multiple constraints may be added.  They will be ORed together.")
     register('--exclude-target-regexp', advanced=True, type=list, default=[],
              metavar='<regexp>',
-             help='Exclude targets that match these regexes.',
-             recursive=True)  # TODO: Does this need to be recursive? What does that even mean?
+             help='Exclude target roots that match these regexes.')
     # Relative pants_distdir to buildroot. Requires --pants-distdir to be bootstrapped above first.
     # e.g. '/dist/'
     rel_distdir = '/{}/'.format(os.path.relpath(register.bootstrap.pants_distdir, get_buildroot()))


Test patch:
diff --git a/tests/python/pants_test/base/BUILD b/tests/python/pants_test/base/BUILD
--- a/tests/python/pants_test/base/BUILD
+++ b/tests/python/pants_test/base/BUILD
@@ -33,8 +33,8 @@ python_tests(
 )
 
 python_tests(
-  name = 'bundle_integration',
-  sources = [ 'test_bundle_integration.py' ],
+  name = 'exclude_target_regexp_integration',
+  sources = [ 'test_exclude_target_regexp_integration.py' ],
   dependencies = [
     'tests/python/pants_test:int-test',
   ],
diff --git a/tests/python/pants_test/base/test_bundle_integration.py b/tests/python/pants_test/base/test_exclude_target_regexp_integration.py
similarity index 88%
rename from tests/python/pants_test/base/test_bundle_integration.py
rename to tests/python/pants_test/base/test_exclude_target_regexp_integration.py
--- a/tests/python/pants_test/base/test_bundle_integration.py
+++ b/tests/python/pants_test/base/test_exclude_target_regexp_integration.py
@@ -40,13 +40,15 @@ def full_spec(self):
   ten_thousand = Bundle('ten-thousand',
       "And now you must face my army of ten thousand BUILD files.")
   there_was_a_duck = Bundle('there-was-a-duck',
-      "And also, there was a duck.")
+      "And also, there was a duck, with her trusty companion.")
+
+  trusty_companion = 'with-her-trusty-companion'
 
   all_bundles = [lesser_of_two, once_upon_a_time, ten_thousand, there_was_a_duck]
 
 
-class BundleIntegrationTest(PantsRunIntegrationTest):
-  """Tests the functionality of the --spec-exclude flag in pants."""
+class ExcludeTargetRegexpIntegrationTest(PantsRunIntegrationTest):
+  """Tests the functionality of the --exclude-target-regexp flag."""
 
   def _bundle_path(self, bundle):
     return os.path.join(get_buildroot(), 'dist',
@@ -135,12 +137,11 @@ def test_exclude_two(self):
     )
 
   @ensure_engine
-  def test_bundle_resource_ordering(self):
-    """Ensures that `resources=` ordering is respected."""
-    pants_run = self.run_pants(
-      ['-q',
-       'run',
-       'testprojects/src/java/org/pantsbuild/testproject/bundle:bundle-resource-ordering']
+  def test_only_exclude_roots(self):
+    # You cannot exclude the trusty companion (ie dependency) of an included root.
+    self._test_bundle_existences([
+          '{}:{}'.format(Bundles.phrase_path, Bundles.there_was_a_duck.spec),
+          '--exclude-target-regexp={}:{}'.format(Bundles.phrase_path, Bundles.trusty_companion),
+        ],
+        set([Bundles.there_was_a_duck]),
     )
-    self.assert_success(pants_run)
-    self.assertEquals(pants_run.stdout_data, 'Hello world from Foo\n\n')
diff --git a/tests/python/pants_test/engine/legacy/BUILD b/tests/python/pants_test/engine/legacy/BUILD
--- a/tests/python/pants_test/engine/legacy/BUILD
+++ b/tests/python/pants_test/engine/legacy/BUILD
@@ -31,7 +31,8 @@ python_tests(
   dependencies = [
     'tests/python/pants_test:int-test'
   ],
-  tags = {'integration'}
+  tags = {'integration'},
+  timeout = 240,
 )
 
 python_tests(
@@ -65,7 +66,7 @@ python_tests(
     'tests/python/pants_test:int-test'
   ],
   tags = {'integration'},
-  timeout = 90,
+  timeout = 30,
 )
 
 python_tests(
diff --git a/tests/python/pants_test/engine/legacy/test_bundle_integration.py b/tests/python/pants_test/engine/legacy/test_bundle_integration.py
--- a/tests/python/pants_test/engine/legacy/test_bundle_integration.py
+++ b/tests/python/pants_test/engine/legacy/test_bundle_integration.py
@@ -54,3 +54,14 @@ def test_bundle_rel_path(self):
         self.assert_success(pants_run)
         self.assertTrue(os.path.isfile(
           '{}/{}.rel_path-bundle/b/file1.txt'.format(temp_distdir, self.TARGET_PATH.replace('/', '.'))))
+
+  @ensure_engine
+  def test_bundle_resource_ordering(self):
+    """Ensures that `resources=` ordering is respected."""
+    pants_run = self.run_pants(
+      ['-q',
+       'run',
+       'testprojects/src/java/org/pantsbuild/testproject/bundle:bundle-resource-ordering']
+    )
+    self.assert_success(pants_run)
+    self.assertEquals(pants_run.stdout_data, 'Hello world from Foo\n\n')
diff --git a/tests/python/pants_test/engine/test_mapper.py b/tests/python/pants_test/engine/test_mapper.py
--- a/tests/python/pants_test/engine/test_mapper.py
+++ b/tests/python/pants_test/engine/test_mapper.py
@@ -54,9 +54,9 @@ class AddressMapTest(unittest.TestCase):
   _symbol_table_cls = ThingTable
 
   @contextmanager
-  def parse_address_map(self, json, exclude_patterns=None):
+  def parse_address_map(self, json):
     path = '/dev/null'
-    address_map = AddressMap.parse(path, json, self._symbol_table_cls, self._parser_cls, exclude_patterns)
+    address_map = AddressMap.parse(path, json, self._symbol_table_cls, self._parser_cls)
     self.assertEqual(path, address_map.path)
     yield address_map
 
@@ -93,77 +93,6 @@ def test_duplicate_names(self):
                                   '{"type_alias": "thing", "name": "one"}'):
         self.fail()
 
-  def test_exclude_target_regexps_plain_string(self):
-    with self.parse_address_map(dedent("""
-      {
-        "type_alias": "thing",
-        "name": "one",
-        "age": 42
-      }
-      {
-        "type_alias": "thing",
-        "name": "two",
-        "age": 37
-      }
-      """), [re.compile('two')]) as address_map:
-      self.assertEqual({'one': Thing(name='one', age=42)}, address_map.objects_by_name)
-
-  def test_exclude_target_regexps_exclude_all(self):
-    with self.parse_address_map(dedent("""
-      {
-        "type_alias": "thing",
-        "name": "one",
-        "age": 42
-      }
-      {
-        "type_alias": "thing",
-        "name": "two",
-        "age": 37
-      }
-      """), [re.compile('o')]) as address_map:
-      self.assertEqual(dict(), address_map.objects_by_name)
-
-  def test_exclude_target_regexps_re_expression(self):
-    with self.parse_address_map(dedent("""
-      {
-        "type_alias": "thing",
-        "name": "one",
-        "age": 42
-      }
-      {
-        "type_alias": "thing",
-        "name": "one_two",
-        "age": 37
-      }
-      {
-        "type_alias": "thing",
-        "name": "one_two_three",
-        "age": 32
-      }
-      """), [re.compile('o.*_two$')]) as address_map:
-      self.assertEqual({'one': Thing(name='one', age=42), 'one_two_three': Thing(name='one_two_three', age=32)},
-                       address_map.objects_by_name)
-
-  def test_exclude_target_regexps_multiple_re(self):
-    with self.parse_address_map(dedent("""
-      {
-        "type_alias": "thing",
-        "name": "one",
-        "age": 42
-      }
-      {
-        "type_alias": "thing",
-        "name": "one_two",
-        "age": 37
-      }
-      {
-        "type_alias": "thing",
-        "name": "one_two_three",
-        "age": 32
-      }
-      """), [re.compile('_.*_'), re.compile('e$')]) as address_map:
-      self.assertEqual({'one_two': Thing(name='one_two', age=37)}, address_map.objects_by_name)
-
 
 class AddressFamilyTest(unittest.TestCase):
   def test_create_single(self):
@@ -305,10 +234,3 @@ def test_walk_descendants_rel_path(self):
                       self.addr('a/d/e:e'): Target(name='e', type_alias='target'),
                       self.addr('a/d/e:e-prime'): Struct(name='e-prime', type_alias='struct')},
                      self.resolve_multi(DescendantAddresses('a/d')))
-
-  # Excludes are not implemented: expects excludes=['a/b', 'a/d/e'].
-  @unittest.expectedFailure
-  def test_walk_descendants_path_excludes(self):
-    self.assertEqual({self.addr('//:root'): Struct(name='root'),
-                      self.addr('a/d:d'): Target(name='d')},
-                     self.resolve_multi(DescendantAddresses('')))
diff --git a/tests/python/pants_test/help/test_help_integration.py b/tests/python/pants_test/help/test_help_integration.py
--- a/tests/python/pants_test/help/test_help_integration.py
+++ b/tests/python/pants_test/help/test_help_integration.py
@@ -47,7 +47,7 @@ def test_help_all_advanced(self):
     self.assertIn('cache.test.junit advanced options:', pants_run.stdout_data)
     # Spot check to see that full args for all options are printed
     self.assertIn('--binary-dup-max-dups', pants_run.stdout_data)
-    self.assertIn('--cache-test-junit-exclude-target-regexp', pants_run.stdout_data)
+    self.assertIn('--cache-test-junit-read', pants_run.stdout_data)
     # Spot check to see that subsystem options are printing
     self.assertIn('--jvm-options', pants_run.stdout_data)
     # Spot check to see that advanced subsystem options are printing