Repository: PrefectHQ/prefect

Issue:
Ability to copy / add files from local to ContainerEnvironments
I'm currently struggling with a Local Secrets issue involving a large JSON blob (specifically, a Google service account key).  Moreover, many other services rely on config files being in known locations.  Instead of wrangling with this, it would be convenient to allow ContainerEnvironments to COPY / ADD local files from the deploying machine to the container.  This could be implemented via simple dictionary:
```python
ContainerEnvironment(files={"/absolute/local/path/filename.ext": "/absolute/container/path/filename.ext"})
```


Discussion:

Resolving patch:
diff --git a/src/prefect/environments.py b/src/prefect/environments.py
--- a/src/prefect/environments.py
+++ b/src/prefect/environments.py
@@ -22,11 +22,11 @@
 """
 
 import base64
+import filecmp
 import json
 import logging
 import os
-import shlex
-import subprocess
+import shutil
 import tempfile
 import textwrap
 import uuid
@@ -219,6 +219,13 @@ class ContainerEnvironment(Environment):
         that will be installed on build of the Docker container
         - image_name (str, optional): A name for the image (usually provided by `build()`)
         - image_tag (str, optional): A tag for the image (usually provided by `build()`)
+        - env_vars (dict, optional): an optional dictionary mapping environment variables to their values (e.g., `{SHELL="bash"}`) to be
+            included in the Dockerfile
+        - files (dict, optional): an optional dictionary mapping local file names to file names in the Docker container; file names should be
+            _absolute paths_.  Note that the COPY directive will be used for these files, so please read the associated Docker documentation.
+
+    Raises:
+        - ValueError: if provided `files` contain non-absolute paths
     """
 
     def __init__(
@@ -228,12 +235,25 @@ def __init__(
         python_dependencies: list = None,
         image_name: str = None,
         image_tag: str = None,
+        env_vars: dict = None,
+        files: dict = None,
     ) -> None:
         self.base_image = base_image
         self.registry_url = registry_url
         self.image_name = image_name
         self.image_tag = image_tag
         self.python_dependencies = python_dependencies or []
+        self.env_vars = env_vars or {}
+        self.files = files or {}
+        not_absolute = [
+            file_path for file_path in self.files if not os.path.isabs(file_path)
+        ]
+        if not_absolute:
+            raise ValueError(
+                "Provided paths {} are not absolute file paths, please provide absolute paths only.".format(
+                    ", ".join(not_absolute)
+                )
+            )
 
     def build(
         self, flow: "prefect.Flow", push: bool = True
@@ -345,6 +365,8 @@ def create_dockerfile(self, flow: "prefect.Flow", directory: str = None) -> None
         In order for the docker python library to build a container it needs a
         Dockerfile that it can use to define the container. This function takes the
         image and python_dependencies then writes them to a file called Dockerfile.
+        
+        *Note*: if `files` are added to this container, they will be copied to this directory as well.
 
         Args:
             - flow (Flow): the flow that the container will run
@@ -363,6 +385,27 @@ def create_dockerfile(self, flow: "prefect.Flow", directory: str = None) -> None
                 for dependency in self.python_dependencies:
                     pip_installs += "RUN pip install {}\n".format(dependency)
 
+            env_vars = ""
+            if self.env_vars:
+                white_space = " " * 20
+                env_vars = "ENV " + " \ \n{}".format(white_space).join(
+                    "{k}={v}".format(k=k, v=v) for k, v in self.env_vars.items()
+                )
+
+            copy_files = ""
+            if self.files:
+                for src, dest in self.files.items():
+                    fname = os.path.basename(src)
+                    if os.path.exists(fname) and filecmp.cmp(src, fname) is False:
+                        raise ValueError(
+                            "File {fname} already exists in {directory}".format(
+                                fname=fname, directory=directory
+                            )
+                        )
+                    else:
+                        shutil.copy2(src, fname)
+                    copy_files += "COPY {fname} {dest}\n".format(fname=fname, dest=dest)
+
             # Create a LocalEnvironment to run the flow
             # the local environment will be placed in the container and run when the container
             # runs
@@ -388,15 +431,19 @@ def create_dockerfile(self, flow: "prefect.Flow", directory: str = None) -> None
                 RUN mkdir /root/.prefect/
                 COPY flow_env.prefect /root/.prefect/flow_env.prefect
                 COPY config.toml /root/.prefect/config.toml
+                {copy_files}
 
                 ENV PREFECT_ENVIRONMENT_FILE="/root/.prefect/flow_env.prefect"
                 ENV PREFECT__USER_CONFIG_PATH="/root/.prefect/config.toml"
+                {env_vars}
 
                 RUN git clone https://{access_token}@github.com/PrefectHQ/prefect.git
                 RUN pip install ./prefect
                 """.format(
                     base_image=self.base_image,
                     pip_installs=pip_installs,
+                    copy_files=copy_files,
+                    env_vars=env_vars,
                     access_token=os.getenv("PERSONAL_ACCESS_TOKEN"),
                 )
             )


Test patch:
diff --git a/tests/test_environments.py b/tests/test_environments.py
--- a/tests/test_environments.py
+++ b/tests/test_environments.py
@@ -57,13 +57,78 @@ def test_create_container_environment(self):
     def test_build_image_process(self):
 
         container = ContainerEnvironment(
-            base_image="python:3.6", tag="tag", registry_url=""
+            base_image="python:3.6", image_tag="tag", registry_url=""
         )
         image = container.build(Flow())
         assert image
 
-        # Need to test running stuff in container, however circleci won't be able to build
-        # a container
+    def test_basic_create_dockerfile(self):
+        container = ContainerEnvironment(base_image="python:3.6", registry_url="")
+        with tempfile.TemporaryDirectory(prefix="prefect-tests") as tmp:
+            container.create_dockerfile(Flow(), directory=tmp)
+            with open(os.path.join(tmp, "Dockerfile"), "r") as f:
+                dockerfile = f.read()
+
+        assert "FROM python:3.6" in dockerfile
+        assert " FROM python:3.6" not in dockerfile
+        assert "RUN pip install ./prefect" in dockerfile
+        assert "RUN mkdir /root/.prefect/" in dockerfile
+        assert "COPY config.toml /root/.prefect/config.toml" in dockerfile
+
+    def test_create_dockerfile_with_environment_variables(self):
+        container = ContainerEnvironment(
+            base_image="python:3.6",
+            registry_url="",
+            env_vars=dict(X=2, Y='"/a/quoted/string/path"'),
+        )
+        with tempfile.TemporaryDirectory(prefix="prefect-tests") as tmp:
+            container.create_dockerfile(Flow(), directory=tmp)
+            with open(os.path.join(tmp, "Dockerfile"), "r") as f:
+                dockerfile = f.read()
+
+        var_orders = [
+            'X=2 \\ \n    Y="/a/quoted/string/path"',
+            'Y="/a/quoted/string/path" \\ \n    X=2',
+        ]
+        assert any(["ENV {}".format(v) in dockerfile for v in var_orders])
+
+    def test_create_dockerfile_with_copy_files(self):
+        with tempfile.NamedTemporaryFile() as t1, tempfile.NamedTemporaryFile() as t2:
+            container = ContainerEnvironment(
+                base_image="python:3.6",
+                registry_url="",
+                files={t1.name: "/root/dockerconfig", t2.name: "./.secret_file"},
+            )
+
+            base1, base2 = os.path.basename(t1.name), os.path.basename(t2.name)
+
+            with tempfile.TemporaryDirectory(prefix="prefect-tests") as tmp:
+                container.create_dockerfile(Flow(), directory=tmp)
+                with open(os.path.join(tmp, "Dockerfile"), "r") as f:
+                    dockerfile = f.read()
+
+        assert "COPY {} /root/dockerconfig".format(base1) in dockerfile
+        assert "COPY {} ./.secret_file".format(base2) in dockerfile
+
+    def test_init_with_copy_files_raises_informative_error_if_not_absolute(self):
+        with pytest.raises(ValueError) as exc:
+            container = ContainerEnvironment(
+                base_image="python:3.6",
+                registry_url="",
+                files={
+                    ".secret_file": "./.secret_file",
+                    "~/.prefect": ".prefect",
+                    "/def/abs": "/def/abs",
+                },
+            )
+
+        file_list = [".secret_file, ~/.prefect", "~/.prefect, .secret_file"]
+        assert any(
+            [
+                "{} are not absolute file paths".format(fs) in str(exc.value)
+                for fs in file_list
+            ]
+        )
 
 
 #################################