Repository: PrefectHQ/prefect

Issue:
RemoteEnvironment doesn't use storage.get_flow() in execute()
Basically, this:

https://github.com/PrefectHQ/prefect/blob/6fe3fc31ce7023403ef0048e7ea1318501391111/src/prefect/environments/execution/remote.py#L68

I thought that an environment would use storage.get_flow() method to retrieve the flow to ensure universal compatibility of different envs/storages.

Not opened as a bug because I'm not sure it is, maybe I'm missing something.


Discussion:
Hi @roveo - interesting catch!  The fix is simple enough, but we also need to implement `get_flow` on the Docker storage object.  This method call would only make sense when called _inside the Docker container itself_.

Resolving patch:
diff --git a/src/prefect/environments/execution/remote.py b/src/prefect/environments/execution/remote.py
--- a/src/prefect/environments/execution/remote.py
+++ b/src/prefect/environments/execution/remote.py
@@ -65,17 +65,13 @@ def execute(  # type: ignore
             )
 
             # Load serialized flow from file and run it with a DaskExecutor
-            with open(flow_location, "rb") as f:
-                flow = cloudpickle.load(f)
+            flow = storage.get_flow(flow_location)
+            with set_temporary_config({"engine.executor.default_class": self.executor}):
+                executor = get_default_executor_class()
 
-                with set_temporary_config(
-                    {"engine.executor.default_class": self.executor}
-                ):
-                    executor = get_default_executor_class()
-
-                executor = executor(**self.executor_kwargs)
-                runner_cls = get_default_flow_runner_class()
-                runner_cls(flow=flow).run(executor=executor)
+            executor = executor(**self.executor_kwargs)
+            runner_cls = get_default_flow_runner_class()
+            runner_cls(flow=flow).run(executor=executor)
         except Exception as exc:
             self.logger.exception(
                 "Unexpected error raised during flow run: {}".format(exc)
diff --git a/src/prefect/environments/storage/docker.py b/src/prefect/environments/storage/docker.py
--- a/src/prefect/environments/storage/docker.py
+++ b/src/prefect/environments/storage/docker.py
@@ -162,6 +162,20 @@ def add_flow(self, flow: "prefect.core.flow.Flow") -> str:
         self._flows[flow.name] = flow  # needed prior to build
         return flow_path
 
+    def get_flow(self, flow_location: str) -> "prefect.core.flow.Flow":
+        """
+        Given a file path within this Docker container, returns the underlying Flow.
+        Note that this method should only be run _within_ the container itself.
+
+        Args:
+            - flow_location (str): the file path of a flow within this container
+
+        Returns:
+            - Flow: the requested flow
+        """
+        with open(flow_location, "rb") as f:
+            return cloudpickle.load(f)
+
     @property
     def name(self) -> str:
         """


Test patch:
diff --git a/tests/environments/execution/test_remote_environment.py b/tests/environments/execution/test_remote_environment.py
--- a/tests/environments/execution/test_remote_environment.py
+++ b/tests/environments/execution/test_remote_environment.py
@@ -35,8 +35,8 @@ def test_environment_execute():
 
         @prefect.task
         def add_to_dict():
-            with open(path.join(directory, "output"), "w") as file:
-                file.write("success")
+            with open(path.join(directory, "output"), "w") as tmp:
+                tmp.write("success")
 
         with open(path.join(directory, "flow_env.prefect"), "w+") as env:
             flow = prefect.Flow("test", tasks=[add_to_dict])
diff --git a/tests/environments/storage/test_docker_storage.py b/tests/environments/storage/test_docker_storage.py
--- a/tests/environments/storage/test_docker_storage.py
+++ b/tests/environments/storage/test_docker_storage.py
@@ -1,3 +1,4 @@
+import cloudpickle
 import os
 import pendulum
 import sys
@@ -442,10 +443,25 @@ def test_docker_storage_name():
     assert storage.name == "test1/test2:test3"
 
 
-def test_docker_storage_doesnt_have_get_flow_method():
+def test_docker_storage_get_flow_method():
     storage = Docker(base_image="python:3.6")
-    with pytest.raises(NotImplementedError):
-        storage.get_flow("")
+    with tempfile.TemporaryDirectory() as directory:
+
+        @prefect.task
+        def add_to_dict():
+            with open(os.path.join(directory, "output"), "w") as tmp:
+                tmp.write("success")
+
+        with open(os.path.join(directory, "flow_env.prefect"), "w+") as env:
+            flow = Flow("test", tasks=[add_to_dict])
+            flow_path = os.path.join(directory, "flow_env.prefect")
+            with open(flow_path, "wb") as f:
+                cloudpickle.dump(flow, f)
+
+        f = storage.get_flow(flow_path)
+        assert isinstance(f, Flow)
+        assert f.name == "test"
+        assert len(f.tasks) == 1
 
 
 def test_add_similar_flows_fails():