Repository: pantsbuild/pants

Issue:
[engine] v2 changed doesn't find linkages to non-`dependencies=` declared deps.
given a jvm_app target like so with a forward dependency edge to `src/java/com/twitter/typeahead:topic-backfill-builder-binary` by way of the `binary=` kwarg:

```
jvm_app(name='topic-backfill-builder-app',
  basename='topic-backfill-builder',
  binary = 'src/java/com/twitter/typeahead:topic-backfill-builder-binary',
  bundles = [
    bundle(fileset=rglobs('config/*', 'data/*', 'scripts/*')),
  ]
)
```

the v2 changed impl fails to detect this target when `src/java/com/twitter/typeahead:topic-backfill-builder-binary` falls in the transitive changed scope - while v1 changed does.

looks like a `traversable_dependency_specs` thing: https://github.com/pantsbuild/pants/blob/master/src/python/pants/backend/jvm/targets/jvm_app.py#L269


Discussion:

Resolving patch:
diff --git a/src/python/pants/bin/engine_initializer.py b/src/python/pants/bin/engine_initializer.py
--- a/src/python/pants/bin/engine_initializer.py
+++ b/src/python/pants/bin/engine_initializer.py
@@ -161,6 +161,6 @@ def setup_legacy_graph(pants_ignore_patterns,
     # TODO: Do not use the cache yet, as it incurs a high overhead.
     scheduler = LocalScheduler(workdir, dict(), tasks, project_tree, native)
     engine = LocalSerialEngine(scheduler, use_cache=False)
-    change_calculator = EngineChangeCalculator(engine, scm) if scm else None
+    change_calculator = EngineChangeCalculator(scheduler, engine, symbol_table_cls, scm) if scm else None
 
     return LegacyGraphHelper(scheduler, engine, symbol_table_cls, change_calculator)
diff --git a/src/python/pants/engine/legacy/change_calculator.py b/src/python/pants/engine/legacy/change_calculator.py
--- a/src/python/pants/engine/legacy/change_calculator.py
+++ b/src/python/pants/engine/legacy/change_calculator.py
@@ -5,94 +5,29 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
-import itertools
 import logging
-from collections import defaultdict
 
 from pants.base.specs import DescendantAddresses
-from pants.build_graph.address import Address
-from pants.engine.legacy.graph import HydratedTargets
-from pants.engine.legacy.source_mapper import EngineSourceMapper, resolve_and_parse_specs
+from pants.engine.legacy.graph import LegacyBuildGraph
+from pants.engine.legacy.source_mapper import EngineSourceMapper
 from pants.scm.change_calculator import ChangeCalculator
 
 
 logger = logging.getLogger(__name__)
 
 
-class _HydratedTargetDependentGraph(object):
-  """A graph for walking dependent addresses of HydratedTarget objects."""
-
-  @classmethod
-  def from_iterable(cls, iterable):
-    """Create a new HydratedTargetDependentGraph from an iterable of HydratedTarget instances."""
-    inst = cls()
-    for hydrated_target in iterable:
-      inst.inject_target(hydrated_target)
-    return inst
-
-  def __init__(self):
-    self._dependent_address_map = defaultdict(set)
-
-  def _resources_addresses(self, hydrated_target):
-    """Yields fully qualified string addresses of resources for a given `HydratedTarget`."""
-    kwargs = hydrated_target.adaptor.kwargs()
-
-    # TODO: Figure out a better way to filter these.
-    # Python targets `resources` are lists of files, not addresses - short circuit for them.
-    if kwargs.get('type_alias', '').startswith('python_'):
-      return
-
-    resource_specs = kwargs.get('resources', [])
-    if not resource_specs:
-      return
-
-    parsed_resource_specs = resolve_and_parse_specs(hydrated_target.adaptor.address.spec_path,
-                                                    resource_specs)
-    for spec in parsed_resource_specs:
-      yield Address.parse(spec.to_spec_string())
-
-  def inject_target(self, hydrated_target):
-    """Inject a target, respecting both its direct dependencies and its resources targets."""
-    for dep in itertools.chain(hydrated_target.dependencies, self._resources_addresses(hydrated_target)):
-      self._dependent_address_map[dep].add(hydrated_target.adaptor.address)
-
-  def dependents_of_addresses(self, addresses):
-    """Given an iterable of addresses, yield all of those addresses dependents."""
-    seen = set(addresses)
-    for address in addresses:
-      for dependent_address in self._dependent_address_map[address]:
-        if dependent_address not in seen:
-          seen.add(dependent_address)
-          yield dependent_address
-
-  def transitive_dependents_of_addresses(self, addresses):
-    """Given an iterable of addresses, yield all of those addresses dependents, transitively."""
-    addresses_to_visit = set(addresses)
-    while 1:
-      dependents = set(self.dependents_of_addresses(addresses))
-      # If we've exhausted all dependencies or visited all remaining nodes, break.
-      if (not dependents) or dependents.issubset(addresses_to_visit):
-        break
-      addresses = dependents.difference(addresses_to_visit)
-      addresses_to_visit.update(dependents)
-
-    transitive_set = itertools.chain(
-      *(self._dependent_address_map[address] for address in addresses_to_visit)
-    )
-    for dep in transitive_set:
-      yield dep
-
-
 class EngineChangeCalculator(ChangeCalculator):
   """A ChangeCalculator variant that uses the v2 engine for source mapping."""
 
-  def __init__(self, engine, scm):
+  def __init__(self, scheduler, engine, symbol_table_cls, scm):
     """
     :param Engine engine: The `Engine` instance to use for computing file to target mappings.
     :param Scm engine: The `Scm` instance to use for computing changes.
     """
     super(EngineChangeCalculator, self).__init__(scm)
+    self._scheduler = scheduler
     self._engine = engine
+    self._symbol_table_cls = symbol_table_cls
     self._mapper = EngineSourceMapper(engine)
 
   def iter_changed_target_addresses(self, changed_request):
@@ -112,17 +47,20 @@ def iter_changed_target_addresses(self, changed_request):
       return
 
     # For dependee finding, we need to parse all build files.
-    product_iter = (t
-                    for targets in self._engine.product_request(HydratedTargets, [DescendantAddresses('')])
-                    for t in targets.dependencies)
-    graph = _HydratedTargetDependentGraph.from_iterable(product_iter)
+    graph = LegacyBuildGraph.create(self._scheduler, self._engine, self._symbol_table_cls)
+    for _ in graph.inject_specs_closure([DescendantAddresses('')]):
+      pass
 
     if changed_request.include_dependees == 'direct':
-      for address in graph.dependents_of_addresses(changed_addresses):
-        yield address
+      emitted = set()
+      for address in changed_addresses:
+        for dependee in graph.dependents_of(address):
+          if dependee not in emitted:
+            emitted.add(dependee)
+            yield dependee
     elif changed_request.include_dependees == 'transitive':
-      for address in graph.transitive_dependents_of_addresses(changed_addresses):
-        yield address
+      for target in graph.transitive_dependees_of_addresses(changed_addresses):
+        yield target.address
 
   def changed_target_addresses(self, changed_request):
     return list(self.iter_changed_target_addresses(changed_request))


Test patch: