Repository: pantsbuild/pants

Issue:
Docker targets can not depend on `shell_source` targets
**Describe the bug**

Given the following files:

`src/docker/project/BUILD`
```
shell_source(
  name='entrypoint',
  source='entrypoint.sh'
)

docker_image(
    repository="example/project",
    dependencies=[
        ":entrypoint",
    ]
)
```

`src/docker/project/entrypoint.sh`
```
#!/usr/bin/bash

echo "Hello"
```

`src/docker/project/Dockerfile`
```
FROM alpine:latest

COPY src/docker/project/entrypoint.sh /bin
CMD ['/bin/entrypoint.sh']
```

When packaging the docker container:

```
$ ./pants build src/docker/project --no-process-execution-local-cleanup 

... <snip> ....
12:12:51.96 [INFO] Preserving local process execution dir /tmp/process-executionSpA0kU for "Building docker image example/project:latest."

... <snip> ....

stderr:
COPY failed: file not found in build context or excluded by .dockerignore: stat: file does not exist

$ find /tmp/process-executionSpA0kU

/tmp/process-executionSpA0kU/src
/tmp/process-executionSpA0kU/src/docker
/tmp/process-executionSpA0kU/src/docker/project
/tmp/process-executionSpA0kU/src/docker/project/Dockerfile
/tmp/process-executionSpA0kU/__run.sh
```

Changing the `shell_source` target to `file` works.


**Pants version**
2.8

**OS**
Linux


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/docker/util_rules/docker_binary.py b/src/python/pants/backend/docker/util_rules/docker_binary.py
--- a/src/python/pants/backend/docker/util_rules/docker_binary.py
+++ b/src/python/pants/backend/docker/util_rules/docker_binary.py
@@ -55,7 +55,7 @@ def build_image(
             argv=tuple(args),
             description=(
                 f"Building docker image {tags[0]}"
-                + (f" +{pluralize(len(tags)-1, 'additional tag')}." if len(tags) > 1 else ".")
+                + (f" +{pluralize(len(tags)-1, 'additional tag')}." if len(tags) > 1 else "")
             ),
             env=env,
             input_digest=digest,
diff --git a/src/python/pants/backend/docker/util_rules/docker_build_context.py b/src/python/pants/backend/docker/util_rules/docker_build_context.py
--- a/src/python/pants/backend/docker/util_rules/docker_build_context.py
+++ b/src/python/pants/backend/docker/util_rules/docker_build_context.py
@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import logging
+from abc import ABC
 from dataclasses import dataclass
 from typing import Mapping
 
@@ -17,6 +18,7 @@
     DockerBuildEnvironment,
     DockerBuildEnvironmentRequest,
 )
+from pants.backend.shell.target_types import ShellSourceField
 from pants.core.goals.package import BuiltPackage, PackageFieldSet
 from pants.core.target_types import FileSourceField
 from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
@@ -28,11 +30,14 @@
     DependenciesRequest,
     FieldSetsPerTarget,
     FieldSetsPerTargetRequest,
+    GeneratedSources,
+    GenerateSourcesRequest,
     SourcesField,
     Targets,
     TransitiveTargets,
     TransitiveTargetsRequest,
 )
+from pants.engine.unions import UnionRule
 from pants.util.frozendict import FrozenDict
 
 logger = logging.getLogger(__name__)
@@ -66,6 +71,41 @@ def merge(self, other: Mapping[str, Mapping[str, str]]) -> DockerVersionContext:
         return DockerVersionContext.from_dict({**self, **other})
 
 
+class DockerContextFilesAcceptableInputsField(ABC, SourcesField):
+    """This is a meta field for the context files generator, to tell the codegen machinery what
+    source fields are good to use.
+
+    Use `DockerContextFilesAcceptableInputsField.register(<SourceField>)` to register input fields
+    that should be accepted.
+
+    This is implemented using the `ABC.register` from Python lib:
+    https://docs.python.org/3/library/abc.html#abc.ABCMeta.register
+    """
+
+
+DockerContextFilesAcceptableInputsField.register(FileSourceField)
+DockerContextFilesAcceptableInputsField.register(ShellSourceField)
+
+
+class DockerContextFilesSourcesField(SourcesField):
+    """This is just a type marker for the codegen machinery."""
+
+
+class GenerateDockerContextFiles(GenerateSourcesRequest):
+    """This translates all files from acceptable Source fields for the docker context using the
+    `codegen` machinery."""
+
+    input = DockerContextFilesAcceptableInputsField
+    output = DockerContextFilesSourcesField
+    exportable = False
+
+
+@rule
+async def hydrate_input_sources(request: GenerateDockerContextFiles) -> GeneratedSources:
+    # We simply pass the files on, as-is
+    return GeneratedSources(request.protocol_sources)
+
+
 @dataclass(frozen=True)
 class DockerBuildContextRequest:
     address: Address
@@ -127,12 +167,13 @@ async def create_docker_build_context(
     # Get all dependencies for the root target.
     root_dependencies = await Get(Targets, DependenciesRequest(docker_image.get(Dependencies)))
 
-    # Get all sources from the root dependencies (i.e. files).
-    file_sources_request = Get(
+    # Get all file sources from the root dependencies. That includes any non-file sources that can
+    # be "codegen"ed into a file source.
+    sources_request = Get(
         SourceFiles,
         SourceFilesRequest(
             sources_fields=[tgt.get(SourcesField) for tgt in root_dependencies],
-            for_sources_types=(FileSourceField,),
+            for_sources_types=(DockerContextFilesSourcesField,),
             enable_codegen=True,
         ),
     )
@@ -142,8 +183,8 @@ async def create_docker_build_context(
         FieldSetsPerTargetRequest(PackageFieldSet, transitive_targets.dependencies),
     )
 
-    file_sources, embedded_pkgs_per_target, dockerfile_info = await MultiGet(
-        file_sources_request,
+    sources, embedded_pkgs_per_target, dockerfile_info = await MultiGet(
+        sources_request,
         embedded_pkgs_per_target_request,
         Get(DockerfileInfo, DockerfileInfoRequest(docker_image.address)),
     )
@@ -161,7 +202,7 @@ async def create_docker_build_context(
     logger.debug(f"Packages for Docker image: {packages_str}")
 
     embedded_pkgs_digest = [built_package.digest for built_package in embedded_pkgs]
-    all_digests = (dockerfile_info.digest, file_sources.snapshot.digest, *embedded_pkgs_digest)
+    all_digests = (dockerfile_info.digest, sources.snapshot.digest, *embedded_pkgs_digest)
 
     # Merge all digests to get the final docker build context digest.
     context_request = Get(Digest, MergeDigests(d for d in all_digests if d))
@@ -180,4 +221,7 @@ async def create_docker_build_context(
 
 
 def rules():
-    return collect_rules()
+    return (
+        *collect_rules(),
+        UnionRule(GenerateSourcesRequest, GenerateDockerContextFiles),
+    )


Test patch:
diff --git a/src/python/pants/backend/docker/util_rules/docker_build_context_test.py b/src/python/pants/backend/docker/util_rules/docker_build_context_test.py
--- a/src/python/pants/backend/docker/util_rules/docker_build_context_test.py
+++ b/src/python/pants/backend/docker/util_rules/docker_build_context_test.py
@@ -23,6 +23,8 @@
 from pants.backend.python.goals.package_pex_binary import PexBinaryFieldSet
 from pants.backend.python.target_types import PexBinary
 from pants.backend.python.util_rules import pex_from_targets
+from pants.backend.shell.target_types import ShellSourcesGeneratorTarget, ShellSourceTarget
+from pants.backend.shell.target_types import rules as shell_target_types_rules
 from pants.core.goals.package import BuiltPackage
 from pants.core.target_types import FilesGeneratorTarget
 from pants.core.target_types import rules as core_target_types_rules
@@ -41,13 +43,20 @@ def rule_runner() -> RuleRunner:
             *package_pex_binary.rules(),
             *parser_rules(),
             *pex_from_targets.rules(),
+            *shell_target_types_rules(),
             *target_types_rules.rules(),
             docker_build_args,
             docker_build_environment_vars,
             QueryRule(BuiltPackage, [PexBinaryFieldSet]),
             QueryRule(DockerBuildContext, (DockerBuildContextRequest,)),
         ],
-        target_types=[DockerImageTarget, FilesGeneratorTarget, PexBinary],
+        target_types=[
+            DockerImageTarget,
+            FilesGeneratorTarget,
+            PexBinary,
+            ShellSourcesGeneratorTarget,
+            ShellSourceTarget,
+        ],
     )
     rule_runner.set_options([], env_inherit={"PATH", "PYENV_ROOT", "HOME"})
     return rule_runner
@@ -248,3 +257,32 @@ def test_synthetic_dockerfile(rule_runner: RuleRunner) -> None:
             "output": {"tag": "1-1"},
         },
     )
+
+
+def test_shell_source_dependencies(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "src/docker/BUILD": dedent(
+                """\
+                docker_image(dependencies=[":entrypoint", ":shell"])
+                shell_source(name="entrypoint", source="entrypoint.sh")
+                shell_sources(name="shell", sources=["scripts/**/*.sh"])
+                """
+            ),
+            "src/docker/Dockerfile": "FROM base",
+            "src/docker/entrypoint.sh": "",
+            "src/docker/scripts/s01.sh": "",
+            "src/docker/scripts/s02.sh": "",
+            "src/docker/scripts/random.file": "",
+        }
+    )
+    assert_build_context(
+        rule_runner,
+        Address("src/docker"),
+        expected_files=[
+            "src/docker/Dockerfile",
+            "src/docker/entrypoint.sh",
+            "src/docker/scripts/s01.sh",
+            "src/docker/scripts/s02.sh",
+        ],
+    )