Repository: pantsbuild/pants

Issue:
Bundle behaviour change for directories in v2
The `v2` engine currently captures matched directories in `Snapshot`s, but doesn't actually propagate them through to `FilesetWithSpec`. One effect of this is the `jvm_app` `bundles` that explicitly refer to directories don't actually end up recursively including the directory content.

The `fileset` args here are matching directories, rather than files, and the semantics of `fileset` are ill defined. The following bundle fileset:
```
bundle(fileset='some/example/directory/')
```
...will not match any files in v2. But changing the glob to:
```
bundle(fileset='some/example/directory/**/*')
```
...will recursively match all files under the directory, similar to the v1 engine.

----

I think there are two changes to make here.

1. We should include all matched paths in the FilesetWithSpec in order to support the existing bundle case, and because it is completely reasonable for `tasks`/`Rules` to be able to capture directories.
2. We should add a deprecation warning for bundles for the case where a bundle matches _only_  directories. The idea is not to ban matching directories in the long term (that's totally reasonable)... instead, the goal would be to try to detect cases where people were expecting everything under a directory to be recursively included by a `fileset`, and instead ask them to change to an explicit recursive glob if that's what they wanted.


Discussion:
Update: it looks like item `1` is not feasible in general, because there are many existing tasks assuming that `FilesetWithSpec` objects only contain files. And the behaviour of `LazyFilesetWithSpec` was that a directory was not included unless it was literally specified.

So I think that I will add a special case to `EagerFilesetWithSpec` creation for bundles to include directories from the snapshot, and put it behind the same deprecation as item `2`.
Would we want to emulate the existing behavior of bundles, or change FileWithSpec's behavior?

Looking at the implementation, Bundle either uses a passed FilesetWithSpec, which only contains file paths, or it uses the strings it was passed with the associated mapper. Then the mapper symlinks those paths.

In v2 do we turn bundles into FilesetWithSpec objects in all cases? We could map filesets with strings into a custom BundleFilesetWithSpec that follows bundles glob rules.
> In v2 do we turn bundles into FilesetWithSpec objects in all cases? We could map filesets with strings into a custom BundleFilesetWithSpec that follows bundles glob rules.

Hm, skipping the creation of `FilesetWithSpec` is a possibility, yea. I think the rest of what I said in my most recent comment still applies.
I agree.

Resolving patch:
diff --git a/src/python/pants/backend/jvm/tasks/bundle_create.py b/src/python/pants/backend/jvm/tasks/bundle_create.py
--- a/src/python/pants/backend/jvm/tasks/bundle_create.py
+++ b/src/python/pants/backend/jvm/tasks/bundle_create.py
@@ -14,7 +14,8 @@
 from pants.backend.jvm.tasks.classpath_products import ClasspathProducts
 from pants.backend.jvm.tasks.jvm_binary_task import JvmBinaryTask
 from pants.base.build_environment import get_buildroot
-from pants.base.exceptions import TaskError
+from pants.base.deprecated import deprecated_conditional
+from pants.base.exceptions import TargetDefinitionException, TaskError
 from pants.build_graph.target_scopes import Scopes
 from pants.fs import archive
 from pants.util.dirutil import absolute_symlink, safe_mkdir, safe_mkdir_for
@@ -208,17 +209,39 @@ def bundle(self, app, results_dir):
         self.shade_jar(shading_rules=app.binary.shading_rules, jar_path=jar_path)
 
     for bundle_counter, bundle in enumerate(app.bundles):
-      fileset_empty = True
+      file_count = 0
+      dir_count = 0
       for path, relpath in bundle.filemap.items():
         bundle_path = os.path.join(bundle_dir, relpath)
-        if os.path.exists(path):
-          safe_mkdir(os.path.dirname(bundle_path))
-          os.symlink(path, bundle_path)
-          fileset_empty = False
-
-      if fileset_empty:
-        raise TaskError('In target {}, bundle index {} of "bundles" field does not match any files.'.format(
-          app.address.spec, bundle_counter))
+        if os.path.isfile(path):
+          file_count += 1
+        elif os.path.isdir(path):
+          dir_count += 1
+        else:
+          continue
+        safe_mkdir(os.path.dirname(bundle_path))
+        os.symlink(path, bundle_path)
+
+      if file_count == 0 and dir_count == 0:
+        raise TargetDefinitionException(app.target,
+                                        'Bundle index {} of "bundles" field '
+                                        'does not match any files.'.format(bundle_counter))
+
+      if dir_count == 1 and file_count == 0:
+        # When this deprecation finishes, we should remove symlinking of directories into the
+        # bundle (which implicitly includes their contents), and instead create them using mkdir.
+        spec = os.path.relpath(bundle.filemap.keys()[0], get_buildroot())
+        deprecated_conditional(
+            lambda: True,
+            '1.5.0.dev0',
+            'default recursive inclusion of files in directory',
+            '''The bundle filespec `{spec}` corresponds to exactly one directory: if you'd like to '''
+            '''continue to recursively include directory contents in future versions, please switch '''
+            '''to a recursive glob like `{fixed_spec}`.'''.format(
+              spec=spec,
+              fixed_spec=os.path.join(spec, '**', '*'),
+            )
+        )
 
     return bundle_dir
 
diff --git a/src/python/pants/engine/legacy/graph.py b/src/python/pants/engine/legacy/graph.py
--- a/src/python/pants/engine/legacy/graph.py
+++ b/src/python/pants/engine/legacy/graph.py
@@ -344,8 +344,9 @@ def hydrate_target(target_adaptor, hydrated_fields):
                         tuple(target_adaptor.dependencies))
 
 
-def _eager_fileset_with_spec(spec_path, filespec, snapshot):
-  files = tuple(fast_relpath(fd.path, spec_path) for fd in snapshot.files)
+def _eager_fileset_with_spec(spec_path, filespec, snapshot, include_dirs=False):
+  fds = snapshot.path_stats if include_dirs else snapshot.files
+  files = tuple(fast_relpath(fd.path, spec_path) for fd in fds)
 
   relpath_adjusted_filespec = FilesetRelPathWrapper.to_filespec(filespec['globs'], spec_path)
   if filespec.has_key('exclude'):
@@ -381,9 +382,13 @@ def hydrate_bundles(bundles_field, snapshot_list):
   for bundle, filespecs, snapshot in zipped:
     spec_path = bundles_field.address.spec_path
     kwargs = bundle.kwargs()
+    # NB: We `include_dirs=True` because bundle filesets frequently specify directories in order
+    # to trigger a (deprecated) default inclusion of their recursive contents. See the related
+    # deprecation in `pants.backend.jvm.tasks.bundle_create`.
     kwargs['fileset'] = _eager_fileset_with_spec(getattr(bundle, 'rel_path', spec_path),
                                                  filespecs,
-                                                 snapshot)
+                                                 snapshot,
+                                                 include_dirs=True)
     bundles.append(BundleAdaptor(**kwargs))
   return HydratedField('bundles', bundles)
 


Test patch:
diff --git a/tests/python/pants_test/engine/legacy/BUILD b/tests/python/pants_test/engine/legacy/BUILD
--- a/tests/python/pants_test/engine/legacy/BUILD
+++ b/tests/python/pants_test/engine/legacy/BUILD
@@ -29,7 +29,8 @@ python_tests(
   name = 'bundle_integration',
   sources = ['test_bundle_integration.py'],
   dependencies = [
-    'tests/python/pants_test:int-test'
+    'src/python/pants/base:deprecated',
+    'tests/python/pants_test:int-test',
   ],
   tags = {'integration'},
   timeout = 240,
diff --git a/tests/python/pants_test/engine/legacy/test_bundle_integration.py b/tests/python/pants_test/engine/legacy/test_bundle_integration.py
--- a/tests/python/pants_test/engine/legacy/test_bundle_integration.py
+++ b/tests/python/pants_test/engine/legacy/test_bundle_integration.py
@@ -7,6 +7,7 @@
 
 import os
 
+from pants.base.deprecated import deprecated_conditional
 from pants.util.contextutil import temporary_dir
 from pants_test.pants_run_integration_test import PantsRunIntegrationTest, ensure_engine
 
@@ -55,6 +56,29 @@ def test_bundle_rel_path(self):
         self.assertTrue(os.path.isfile(
           '{}/{}.rel_path-bundle/b/file1.txt'.format(temp_distdir, self.TARGET_PATH.replace('/', '.'))))
 
+  @ensure_engine
+  def test_bundle_directory(self):
+    with temporary_dir() as temp_distdir:
+      with self.pants_results(
+        ['-q',
+          '--pants-distdir={}'.format(temp_distdir),
+          'bundle',
+          '{}:directory'.format(self.TARGET_PATH)]) as pants_run:
+        self.assert_success(pants_run)
+
+        root = '{}/{}.directory-bundle/a/b'.format(temp_distdir, self.TARGET_PATH.replace('/', '.'))
+
+        self.assertTrue(os.path.isdir(root))
+
+        # NB: The behaviour of this test will change with the relevant deprecation
+        # in `pants.backend.jvm.tasks.bundle_create`.
+        deprecated_conditional(
+            lambda: os.path.isfile(os.path.join(root, 'file1.txt')),
+            '1.5.0.dev0',
+            'default recursive inclusion of files in directory',
+            'A non-recursive/literal glob should no longer include child paths.'
+        )
+
   @ensure_engine
   def test_bundle_resource_ordering(self):
     """Ensures that `resources=` ordering is respected."""