Repository: pantsbuild/pants

Issue:
Bundle behaviour change for directories in v2
The `v2` engine currently captures matched directories in `Snapshot`s, but doesn't actually propagate them through to `FilesetWithSpec`. One effect of this is the `jvm_app` `bundles` that explicitly refer to directories don't actually end up recursively including the directory content.

The `fileset` args here are matching directories, rather than files, and the semantics of `fileset` are ill defined. The following bundle fileset:
```
bundle(fileset='some/example/directory/')
```
...will not match any files in v2. But changing the glob to:
```
bundle(fileset='some/example/directory/**/*')
```
...will recursively match all files under the directory, similar to the v1 engine.

----

I think there are two changes to make here.

1. We should include all matched paths in the FilesetWithSpec in order to support the existing bundle case, and because it is completely reasonable for `tasks`/`Rules` to be able to capture directories.
2. We should add a deprecation warning for bundles for the case where a bundle matches _only_  directories. The idea is not to ban matching directories in the long term (that's totally reasonable)... instead, the goal would be to try to detect cases where people were expecting everything under a directory to be recursively included by a `fileset`, and instead ask them to change to an explicit recursive glob if that's what they wanted.


Discussion:
Update: it looks like item `1` is not feasible in general, because there are many existing tasks assuming that `FilesetWithSpec` objects only contain files. And the behaviour of `LazyFilesetWithSpec` was that a directory was not included unless it was literally specified.

So I think that I will add a special case to `EagerFilesetWithSpec` creation for bundles to include directories from the snapshot, and put it behind the same deprecation as item `2`.
Would we want to emulate the existing behavior of bundles, or change FileWithSpec's behavior?

Looking at the implementation, Bundle either uses a passed FilesetWithSpec, which only contains file paths, or it uses the strings it was passed with the associated mapper. Then the mapper symlinks those paths.

In v2 do we turn bundles into FilesetWithSpec objects in all cases? We could map filesets with strings into a custom BundleFilesetWithSpec that follows bundles glob rules.
> In v2 do we turn bundles into FilesetWithSpec objects in all cases? We could map filesets with strings into a custom BundleFilesetWithSpec that follows bundles glob rules.

Hm, skipping the creation of `FilesetWithSpec` is a possibility, yea. I think the rest of what I said in my most recent comment still applies.
I agree.

Resolving patch:
diff --git a/src/python/pants/backend/jvm/tasks/bundle_create.py b/src/python/pants/backend/jvm/tasks/bundle_create.py
--- a/src/python/pants/backend/jvm/tasks/bundle_create.py
+++ b/src/python/pants/backend/jvm/tasks/bundle_create.py
@@ -14,7 +14,8 @@
 from pants.backend.jvm.tasks.classpath_products import ClasspathProducts
 from pants.backend.jvm.tasks.jvm_binary_task import JvmBinaryTask
 from pants.base.build_environment import get_buildroot
-from pants.base.exceptions import TaskError
+from pants.base.deprecated import deprecated_conditional
+from pants.base.exceptions import TargetDefinitionException, TaskError
 from pants.build_graph.target_scopes import Scopes
 from pants.fs import archive
 from pants.util.dirutil import absolute_symlink, safe_mkdir, safe_mkdir_for
@@ -207,21 +208,57 @@ def bundle(self, app, results_dir):
         # TODO run in parallel to speed up
         self.shade_jar(shading_rules=app.binary.shading_rules, jar_path=jar_path)
 
-    for bundle_counter, bundle in enumerate(app.bundles):
-      fileset_empty = True
-      for path, relpath in bundle.filemap.items():
-        bundle_path = os.path.join(bundle_dir, relpath)
-        if os.path.exists(path):
-          safe_mkdir(os.path.dirname(bundle_path))
-          os.symlink(path, bundle_path)
-          fileset_empty = False
-
-      if fileset_empty:
-        raise TaskError('In target {}, bundle index {} of "bundles" field does not match any files.'.format(
-          app.address.spec, bundle_counter))
+    self._symlink_bundles(app, bundle_dir)
 
     return bundle_dir
 
+  def _symlink_bundles(self, app, bundle_dir):
+    """For each bundle in the given app, symlinks relevant matched paths from shortest to longest.
+
+    Validates that at least one path was matched by a bundle, and (temporarily: see the
+    deprecation) symlinks matched directories to recursively include their contents.
+    """
+    for bundle_counter, bundle in enumerate(app.bundles):
+      file_count = 0
+      dir_count = 0
+      # Create in ascending path-length order to ensure that symlinks to directories
+      # are created before their contents would be. Can remove ordering along with the
+      # 'recursive inclusion' deprecation (when only files and not directories are
+      # symlinked).
+      for path, relpath in sorted(bundle.filemap.items(), key=lambda e: len(e[0])):
+        bundle_path = os.path.join(bundle_dir, relpath)
+        if os.path.isfile(path):
+          file_count += 1
+        elif os.path.isdir(path):
+          dir_count += 1
+        else:
+          continue
+        if os.path.exists(bundle_path):
+          continue
+        safe_mkdir(os.path.dirname(bundle_path))
+        os.symlink(path, bundle_path)
+
+      if file_count == 0 and dir_count == 0:
+        raise TargetDefinitionException(app.target,
+                                        'Bundle index {} of "bundles" field '
+                                        'does not match any files.'.format(bundle_counter))
+
+      if dir_count == 1 and file_count == 0:
+        # When this deprecation finishes, we should remove symlinking of directories into the
+        # bundle (which implicitly includes their contents), and instead create them using mkdir.
+        spec = os.path.relpath(bundle.filemap.keys()[0], get_buildroot())
+        deprecated_conditional(
+            lambda: True,
+            '1.5.0.dev0',
+            'default recursive inclusion of files in directory',
+            'The bundle filespec `{spec}` corresponds to exactly one directory: if you\'d like to '
+            'continue to recursively include directory contents in future versions, please switch '
+            'to a recursive glob like `{fixed_spec}`.'.format(
+              spec=spec,
+              fixed_spec=os.path.join(spec, '**', '*'),
+            )
+        )
+
   def check_basename_conflicts(self, targets):
     """Apps' basenames are used as bundle directory names. Ensure they are all unique."""
 
diff --git a/src/python/pants/engine/legacy/graph.py b/src/python/pants/engine/legacy/graph.py
--- a/src/python/pants/engine/legacy/graph.py
+++ b/src/python/pants/engine/legacy/graph.py
@@ -344,8 +344,9 @@ def hydrate_target(target_adaptor, hydrated_fields):
                         tuple(target_adaptor.dependencies))
 
 
-def _eager_fileset_with_spec(spec_path, filespec, snapshot):
-  files = tuple(fast_relpath(fd.path, spec_path) for fd in snapshot.files)
+def _eager_fileset_with_spec(spec_path, filespec, snapshot, include_dirs=False):
+  fds = snapshot.path_stats if include_dirs else snapshot.files
+  files = tuple(fast_relpath(fd.path, spec_path) for fd in fds)
 
   relpath_adjusted_filespec = FilesetRelPathWrapper.to_filespec(filespec['globs'], spec_path)
   if filespec.has_key('exclude'):
@@ -381,9 +382,13 @@ def hydrate_bundles(bundles_field, snapshot_list):
   for bundle, filespecs, snapshot in zipped:
     spec_path = bundles_field.address.spec_path
     kwargs = bundle.kwargs()
+    # NB: We `include_dirs=True` because bundle filesets frequently specify directories in order
+    # to trigger a (deprecated) default inclusion of their recursive contents. See the related
+    # deprecation in `pants.backend.jvm.tasks.bundle_create`.
     kwargs['fileset'] = _eager_fileset_with_spec(getattr(bundle, 'rel_path', spec_path),
                                                  filespecs,
-                                                 snapshot)
+                                                 snapshot,
+                                                 include_dirs=True)
     bundles.append(BundleAdaptor(**kwargs))
   return HydratedField('bundles', bundles)
 


Test patch:
diff --git a/tests/python/pants_test/engine/legacy/BUILD b/tests/python/pants_test/engine/legacy/BUILD
--- a/tests/python/pants_test/engine/legacy/BUILD
+++ b/tests/python/pants_test/engine/legacy/BUILD
@@ -29,7 +29,8 @@ python_tests(
   name = 'bundle_integration',
   sources = ['test_bundle_integration.py'],
   dependencies = [
-    'tests/python/pants_test:int-test'
+    'src/python/pants/base:deprecated',
+    'tests/python/pants_test:int-test',
   ],
   tags = {'integration'},
   timeout = 240,
diff --git a/tests/python/pants_test/engine/legacy/test_bundle_integration.py b/tests/python/pants_test/engine/legacy/test_bundle_integration.py
--- a/tests/python/pants_test/engine/legacy/test_bundle_integration.py
+++ b/tests/python/pants_test/engine/legacy/test_bundle_integration.py
@@ -6,7 +6,9 @@
                         unicode_literals, with_statement)
 
 import os
+from contextlib import contextmanager
 
+from pants.base.deprecated import deprecated_conditional
 from pants.util.contextutil import temporary_dir
 from pants_test.pants_run_integration_test import PantsRunIntegrationTest, ensure_engine
 
@@ -19,41 +21,53 @@ def test_bundle_basic(self):
     args = ['-q', 'bundle', self.TARGET_PATH]
     self.do_command(*args, success=True, enable_v2_engine=True)
 
-  @ensure_engine
-  def test_bundle_mapper(self):
+  @contextmanager
+  def bundled(self, target_name):
     with temporary_dir() as temp_distdir:
       with self.pants_results(
           ['-q',
            '--pants-distdir={}'.format(temp_distdir),
            'bundle',
-           '{}:mapper'.format(self.TARGET_PATH)]) as pants_run:
+           '{}:{}'.format(self.TARGET_PATH, target_name)]) as pants_run:
         self.assert_success(pants_run)
-        self.assertTrue(os.path.isfile(
-          '{}/{}.mapper-bundle/bundle_files/file1.txt'.format(temp_distdir, self.TARGET_PATH.replace('/', '.'))))
+        yield os.path.join(temp_distdir,
+                           '{}.{}-bundle'.format(self.TARGET_PATH.replace('/', '.'), target_name))
+
+  @ensure_engine
+  def test_bundle_mapper(self):
+    with self.bundled('mapper') as bundle_dir:
+      self.assertTrue(os.path.isfile(os.path.join(bundle_dir, 'bundle_files/file1.txt')))
 
   @ensure_engine
   def test_bundle_relative_to(self):
-    with temporary_dir() as temp_distdir:
-      with self.pants_results(
-        ['-q',
-          '--pants-distdir={}'.format(temp_distdir),
-          'bundle',
-          '{}:relative_to'.format(self.TARGET_PATH)]) as pants_run:
-        self.assert_success(pants_run)
-        self.assertTrue(os.path.isfile(
-          '{}/{}.relative_to-bundle/b/file1.txt'.format(temp_distdir, self.TARGET_PATH.replace('/', '.'))))
+    with self.bundled('relative_to') as bundle_dir:
+      self.assertTrue(os.path.isfile(os.path.join(bundle_dir, 'b/file1.txt')))
 
   @ensure_engine
   def test_bundle_rel_path(self):
-    with temporary_dir() as temp_distdir:
-      with self.pants_results(
-        ['-q',
-          '--pants-distdir={}'.format(temp_distdir),
-          'bundle',
-          '{}:rel_path'.format(self.TARGET_PATH)]) as pants_run:
-        self.assert_success(pants_run)
-        self.assertTrue(os.path.isfile(
-          '{}/{}.rel_path-bundle/b/file1.txt'.format(temp_distdir, self.TARGET_PATH.replace('/', '.'))))
+    with self.bundled('rel_path') as bundle_dir:
+      self.assertTrue(os.path.isfile(os.path.join(bundle_dir, 'b/file1.txt')))
+
+  @ensure_engine
+  def test_bundle_directory(self):
+    with self.bundled('directory') as bundle_dir:
+      root = os.path.join(bundle_dir, 'a/b')
+      self.assertTrue(os.path.isdir(root))
+      # NB: The behaviour of this test will change with the relevant deprecation
+      # in `pants.backend.jvm.tasks.bundle_create`, because the parent directory
+      # will not be symlinked.
+      deprecated_conditional(
+          lambda: os.path.isfile(os.path.join(root, 'file1.txt')),
+          '1.5.0.dev0',
+          'default recursive inclusion of files in directory',
+          'A non-recursive/literal glob should no longer include child paths.'
+      )
+
+  def test_bundle_explicit_recursion(self):
+    with self.bundled('explicit_recursion') as bundle_dir:
+      root = os.path.join(bundle_dir, 'a/b')
+      self.assertTrue(os.path.isdir(root))
+      self.assertTrue(os.path.isfile(os.path.join(root, 'file1.txt')))
 
   @ensure_engine
   def test_bundle_resource_ordering(self):