Repository: PrefectHQ/prefect

Issue:
"Bad request for URL" when running "prefect get flow-runs"
## Description
I see the following error when running `prefect get flow-runs`:
```
Traceback (most recent call last):
  File "/home/jc/.local/bin/prefect", line 10, in <module>
    sys.exit(cli())
  File "/usr/local/lib/python3.7/dist-packages/click/core.py", line 829, in __call__
    return self.main(*args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/click/core.py", line 782, in main
    rv = self.invoke(ctx)
  File "/usr/local/lib/python3.7/dist-packages/click/core.py", line 1259, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/usr/local/lib/python3.7/dist-packages/click/core.py", line 1259, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/usr/local/lib/python3.7/dist-packages/click/core.py", line 1066, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/usr/local/lib/python3.7/dist-packages/click/core.py", line 610, in invoke
    return callback(*args, **kwargs)
  File "/home/jc/.local/lib/python3.7/site-packages/prefect/cli/get.py", line 256, in flow_runs
    result = Client().graphql(query)
  File "/home/jc/.local/lib/python3.7/site-packages/prefect/client/client.py", line 281, in graphql
    retry_on_api_error=retry_on_api_error,
  File "/home/jc/.local/lib/python3.7/site-packages/prefect/client/client.py", line 237, in post
    retry_on_api_error=retry_on_api_error,
  File "/home/jc/.local/lib/python3.7/site-packages/prefect/client/client.py", line 401, in _request
    session=session, method=method, url=url, params=params, headers=headers
  File "/home/jc/.local/lib/python3.7/site-packages/prefect/client/client.py", line 333, in _send_request
    response.raise_for_status()
  File "/home/jc/.local/lib/python3.7/site-packages/requests/models.py", line 941, in raise_for_status
    raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http://localhost:4200/graphql
```

The same error occurs if I specify a flow (`--flow`) or project (`--project`). 

I'm running 0.13.11. The version of prefect matches the tags of the running prefect containers.

## Reproduction
On a VM:
```
pip3 install prefect
prefect backend server
prefect server start
```
Wait for images to download and the services to start. Then run:
```
prefect get flow-runs
```



Comments:

Patch:
diff --git a/changes/issue3517.yaml b/changes/issue3517.yaml
new file mode 100644
index 0000000000..f0e161d288
--- /dev/null
+++ b/changes/issue3517.yaml
@@ -0,0 +1,2 @@
+fix:
+  - "Fix `get flow-runs` and `describe flow-runs` CLI commands querying of removed `duration` field - [#3517](https://github.com/PrefectHQ/prefect/issues/3517)"
diff --git a/changes/pr3511.yaml b/changes/pr3511.yaml
new file mode 100644
index 0000000000..9389fc944c
--- /dev/null
+++ b/changes/pr3511.yaml
@@ -0,0 +1,2 @@
+fix:
+  - "Fix multiprocessing scheduler failure while running tasks with timeouts - [#3511](https://github.com/PrefectHQ/prefect/pull/3511)"
diff --git a/docs/core/task_library/overview.md b/docs/core/task_library/overview.md
index f1c02ea2a5..1d84f2b59b 100644
--- a/docs/core/task_library/overview.md
+++ b/docs/core/task_library/overview.md
@@ -19,7 +19,7 @@ for the `prefect.tasks` module.
 | <img src="/logos/dbt.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>DBT</p>](https://docs.prefect.io/api/latest/tasks/dbt.html) | <img src="/logos/docker.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Docker</p>](https://docs.prefect.io/api/latest/tasks/docker.html) | <img src="/logos/dropbox.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Dropbox</p>](https://docs.prefect.io/api/latest/tasks/dropbox.html) | <img src="/logos/email.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Email</p>](https://docs.prefect.io/api/latest/tasks/email.html) | <img src="/logos/github.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>GitHub</p>](https://docs.prefect.io/api/latest/tasks/github.html) |
 | <img src="/logos/google_cloud.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Google Cloud</p>](https://docs.prefect.io/api/latest/tasks/gcp.html) | <img src="/logos/sheets.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Google Sheets</p>](https://docs.prefect.io/api/latest/tasks/google_sheets.html) | <img src="/logos/ge.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Great Expectations</p>](https://docs.prefect.io/api/latest/tasks/great_expectations.html) | <img src="/logos/jira.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Jira</p>](https://docs.prefect.io/api/latest/tasks/jira.html) | <img src="/logos/kubernetes.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Kubernetes</p>](https://docs.prefect.io/api/latest/tasks/kubernetes.html) |
 | <img src="/logos/mysql.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>MySQL</p>](https://docs.prefect.io/api/latest/tasks/mysql.html) | <img src="/logos/monday.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Monday</p>](https://docs.prefect.io/api/latest/tasks/monday.html) | <img src="/logos/postgres.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>PostgreSQL</p>](https://docs.prefect.io/api/latest/tasks/postgres.html) | <img src="/logos/python.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Python</p>](https://docs.prefect.io/api/latest/tasks/function.html) | <img src="/logos/pushbullet.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Pushbullet</p>](https://docs.prefect.io/api/latest/tasks/pushbullet.html) |
-| <img src="/logos/redis.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Redis</p>](https://docs.prefect.io/api/latest/tasks/redis.html) | <img src="/logos/rlogo.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>RSS</p>](https://docs.prefect.io/api/latest/tasks/rss.html) | <img src="/logos/shell.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Shell</p>](https://docs.prefect.io/api/latest/tasks/shell.html) | <img src="/logos/slack.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Slack</p>](https://docs.prefect.io/api/latest/tasks/slack.html)| <img src="/logos/snowflake.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Snowflake</p>](https://docs.prefect.io/api/latest/tasks/snowflake.html) |
+| <img src="/logos/redis.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Redis</p>](https://docs.prefect.io/api/latest/tasks/redis.html) | <img src="/logos/rlogo.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>RSS</p>](https://docs.prefect.io/api/latest/tasks/rss.html) | <img src="/logos/shell.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Shell</p>](https://docs.prefect.io/api/latest/tasks/shell.html) | <img src="/logos/slack.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Slack</p>](https://docs.prefect.io/api/latest/tasks/notifications.html#slacktask)| <img src="/logos/snowflake.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Snowflake</p>](https://docs.prefect.io/api/latest/tasks/snowflake.html) |
 | <img src="/logos/spacy.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>SpaCy</p>](https://docs.prefect.io/api/latest/tasks/spacy.html) | <img src="/logos/sqlite.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>SQLite</p>](https://docs.prefect.io/api/latest/tasks/sqlite.html) | <img src="/logos/tlogo.png" height=128 width=128 style="max-height: 128px; max-width: 128px;"> [<p>Twitter</p>](https://docs.prefect.io/api/latest/tasks/twitter.html) |
 
 ## Task library in action
diff --git a/src/prefect/cli/describe.py b/src/prefect/cli/describe.py
index 7fc7cdd3c1..32150765a2 100644
--- a/src/prefect/cli/describe.py
+++ b/src/prefect/cli/describe.py
@@ -239,7 +239,6 @@ def flow_runs(name, flow_name, output):
                 "scheduled_start_time": True,
                 "start_time": True,
                 "end_time": True,
-                "duration": True,
                 "serialized_state": True,
             }
         }
diff --git a/src/prefect/cli/get.py b/src/prefect/cli/get.py
index f908f10cb5..f81ef7919b 100644
--- a/src/prefect/cli/get.py
+++ b/src/prefect/cli/get.py
@@ -247,7 +247,6 @@ def flow_runs(limit, flow, project, started):
                 "created": True,
                 "state": True,
                 "name": True,
-                "duration": True,
                 "start_time": True,
             }
         }
@@ -271,7 +270,6 @@ def flow_runs(limit, flow, project, started):
                 item.state,
                 pendulum.parse(item.created).diff_for_humans(),
                 start_time,
-                item.duration,
                 item.id,
             ]
         )
@@ -285,7 +283,6 @@ def flow_runs(limit, flow, project, started):
                 "STATE",
                 "AGE",
                 "START TIME",
-                "DURATION",
                 "ID",
             ],
             tablefmt="plain",
diff --git a/src/prefect/utilities/executors.py b/src/prefect/utilities/executors.py
index bc2a189b53..2c0027b75c 100644
--- a/src/prefect/utilities/executors.py
+++ b/src/prefect/utilities/executors.py
@@ -10,7 +10,9 @@ import warnings
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures import TimeoutError as FutureTimeout
 from functools import wraps
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Union, Sequence
+
+import cloudpickle
 
 import prefect
 
@@ -108,20 +110,68 @@ def main_thread_timeout(
         signal.alarm(0)
 
 
+def multiprocessing_safe_retrieve_value(
+    queue: multiprocessing.Queue,
+    payload: bytes,
+) -> None:
+    """
+    Gets the return value from a function and puts it in a multiprocessing-safe
+    container. Helper function for `multiprocessing_timeout`, must be defined top-level
+    so it can be pickled and sent to `multiprocessing.Process`
+
+    Passing the payload serialized allows us to escape the limitations of the python
+    native pickler which will fail on tasks defined in scripts because of name
+    mismatches. Whilst this particular example only affects the `func` arg, any of the
+    others could be affected by other pickle limitations as well.
+
+    Args:
+        - queue (multiprocessing.Queue): The queue to pass the resulting payload to
+        - payload (bytes): A serialized dictionary containing the data required to run
+            the function. Should be serialized with `cloudpickle.dumps`
+            Expects the following keys:
+            - fn (Callable): The function to call
+            - args (list): Positional argument values to call the function with
+            - kwargs (dict): Keyword arguments to call the function with
+            - context (dict): The prefect context dictionary to use during execution
+
+    Returns:
+        - None
+        Passes the serialized (with cloudpickle) return value or exception into the
+        queue. Callers are expected to re-raise any exceptions.
+    """
+    request = cloudpickle.loads(payload)
+
+    fn: Callable = request["fn"]
+    context: dict = request.get("context", {})
+    args: Sequence = request.get("args", [])
+    kwargs: dict = request.get("kwargs", {})
+
+    try:
+        with prefect.context(context):
+            return_val = fn(*args, **kwargs)
+    except Exception as exc:
+        return_val = exc
+
+    queue.put(cloudpickle.dumps(return_val))
+
+
 def multiprocessing_timeout(
     fn: Callable, *args: Any, timeout: int = None, **kwargs: Any
 ) -> Any:
     """
     Helper function for implementing timeouts on function executions.
     Implemented by spawning a new multiprocess.Process() and joining with timeout.
+
     Args:
         - fn (callable): the function to execute
         - *args (Any): arguments to pass to the function
-        - timeout (int): the length of time to allow for
-            execution before raising a `TimeoutError`, represented as an integer in seconds
+        - timeout (int): the length of time to allow for execution before raising a
+            `TimeoutError`, represented as an integer in seconds
         - **kwargs (Any): keyword arguments to pass to the function
+
     Returns:
         - the result of `f(*args, **kwargs)`
+
     Raises:
         - AssertionError: if run from a daemonic process
         - TimeoutError: if function execution exceeds the allowed timeout
@@ -130,26 +180,29 @@ def multiprocessing_timeout(
     if timeout is None:
         return fn(*args, **kwargs)
 
-    def retrieve_value(
-        *args: Any, _container: multiprocessing.Queue, _ctx_dict: dict, **kwargs: Any
-    ) -> None:
-        """Puts the return value in a multiprocessing-safe container"""
-        try:
-            with prefect.context(_ctx_dict):
-                val = fn(*args, **kwargs)
-            _container.put(val)
-        except Exception as exc:
-            _container.put(exc)
-
-    q = multiprocessing.Queue()  # type: multiprocessing.Queue
-    kwargs["_container"] = q
-    kwargs["_ctx_dict"] = prefect.context.to_dict()
-    p = multiprocessing.Process(target=retrieve_value, args=args, kwargs=kwargs)
+    # Create a queue to pass the function return value back
+    queue = multiprocessing.Queue()  # type: multiprocessing.Queue
+
+    # Set internal kwargs for the helper function
+    request = {
+        "fn": fn,
+        "args": args,
+        "kwargs": kwargs,
+        "context": prefect.context.to_dict(),
+    }
+    payload = cloudpickle.dumps(request)
+
+    p = multiprocessing.Process(
+        target=multiprocessing_safe_retrieve_value, args=(queue, payload)
+    )
     p.start()
     p.join(timeout)
     p.terminate()
-    if not q.empty():
-        res = q.get()
+
+    # Handle the process result, if the queue is empty the function did not finish
+    # before the timeout
+    if not queue.empty():
+        res = cloudpickle.loads(queue.get())
         if isinstance(res, Exception):
             raise res
         return res
@@ -173,8 +226,8 @@ def timeout_handler(
     Args:
         - fn (callable): the function to execute
         - *args (Any): arguments to pass to the function
-        - timeout (int): the length of time to allow for
-            execution before raising a `TimeoutError`, represented as an integer in seconds
+        - timeout (int): the length of time to allow for execution before raising a
+            `TimeoutError`, represented as an integer in seconds
         - **kwargs (Any): keyword arguments to pass to the function
 
     Returns:
@@ -196,19 +249,16 @@ def timeout_handler(
         elif multiprocessing.current_process().daemon is False:
             return multiprocessing_timeout(fn, *args, timeout=timeout, **kwargs)
 
-        msg = (
-            "This task is running in a daemonic subprocess; "
-            "consequently Prefect can only enforce a soft timeout limit, i.e., "
-            "if your Task reaches its timeout limit it will enter a TimedOut state "
-            "but continue running in the background."
-        )
+        soft_timeout_reason = "in a daemonic subprocess"
     else:
-        msg = (
-            "This task is running on Windows; "
-            "consequently Prefect can only enforce a soft timeout limit, i.e., "
-            "if your Task reaches its timeout limit it will enter a TimedOut state "
-            "but continue running in the background."
-        )
+        soft_timeout_reason = "on Windows"
+
+    msg = (
+        f"This task is running {soft_timeout_reason}; "
+        "consequently Prefect can only enforce a soft timeout limit, i.e., "
+        "if your Task reaches its timeout limit it will enter a TimedOut state "
+        "but continue running in the background."
+    )
 
     warnings.warn(msg, stacklevel=2)
     executor = ThreadPoolExecutor()
@@ -224,7 +274,10 @@ def timeout_handler(
     try:
         return fut.result(timeout=timeout)
     except FutureTimeout as exc:
-        raise TimeoutError("Execution timed out.") from exc
+        raise TimeoutError(
+            f"Execution timed out but was executed {soft_timeout_reason} and will "
+            "continue to run in the background."
+        ) from exc
 
 
 class RecursiveCall(Exception):


Test patch:
diff --git a/tests/cli/test_describe.py b/tests/cli/test_describe.py
index b06dfb87ba..a56c746142 100644
--- a/tests/cli/test_describe.py
+++ b/tests/cli/test_describe.py
@@ -263,7 +263,6 @@ def test_describe_flow_runs(monkeypatch, cloud_api):
             scheduled_start_time
             start_time
             end_time
-            duration
             serialized_state
         }
     }
@@ -320,7 +319,6 @@ def test_describe_flow_runs_populated(monkeypatch, cloud_api):
             scheduled_start_time
             start_time
             end_time
-            duration
             serialized_state
         }
     }
diff --git a/tests/cli/test_get.py b/tests/cli/test_get.py
index 70fcef9541..93edc3be0f 100644
--- a/tests/cli/test_get.py
+++ b/tests/cli/test_get.py
@@ -195,7 +195,6 @@ def test_get_flow_runs_cloud(monkeypatch, cloud_api):
             and "STATE" in result.output
             and "AGE" in result.output
             and "START TIME" in result.output
-            and "DURATION" in result.output
         )
 
         query = """
@@ -208,7 +207,6 @@ def test_get_flow_runs_cloud(monkeypatch, cloud_api):
                 created
                 state
                 name
-                duration
                 start_time
             }
         }
@@ -255,7 +253,6 @@ def test_get_flow_runs_populated(monkeypatch, cloud_api):
                 created
                 state
                 name
-                duration
                 start_time
             }
         }
diff --git a/tests/conftest.py b/tests/conftest.py
index ad70a37d9c..4f582e8562 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -35,7 +35,7 @@ def prefect_home_dir():
 # ----------------
 @pytest.fixture(scope="session")
 def mthread():
-    "Multi-threaded executor"
+    "Multi-threaded executor using dask distributed"
     with Client(
         processes=False, scheduler_port=0, dashboard_address=":0", n_workers=2
     ) as client:
@@ -51,12 +51,18 @@ def local():
 @pytest.fixture()
 def sync():
     "Synchronous dask (not dask.distributed) executor"
-    yield LocalDaskExecutor()
+    yield LocalDaskExecutor(scheduler="sync")
+
+
+@pytest.fixture()
+def mproc_local():
+    "Multiprocessing executor using local dask (not distributed cluster)"
+    yield LocalDaskExecutor(scheduler="processes")
 
 
 @pytest.fixture(scope="session")
 def mproc():
-    "Multi-processing executor"
+    "Multi-processing executor using dask distributed"
     with Client(
         processes=True, scheduler_port=0, dashboard_address=":0", n_workers=2
     ) as client:
@@ -64,14 +70,16 @@ def mproc():
 
 
 @pytest.fixture()
-def _switch(mthread, local, sync, mproc):
+def _switch(mthread, local, sync, mproc, mproc_local):
     """
     A construct needed so we can parametrize the executor fixture.
 
     This isn't straightforward since each executor needs to be initialized
     in slightly different ways.
     """
-    execs = dict(mthread=mthread, local=local, sync=sync, mproc=mproc)
+    execs = dict(
+        mthread=mthread, local=local, sync=sync, mproc=mproc, mproc_local=mproc_local
+    )
     return lambda e: execs[e]
 
 
diff --git a/tests/core/test_flow.py b/tests/core/test_flow.py
index 9460bf4e9f..6b763a07ae 100644
--- a/tests/core/test_flow.py
+++ b/tests/core/test_flow.py
@@ -22,7 +22,7 @@ from prefect.core.task import Task
 from prefect.tasks.core import constants
 from prefect.core.parameter import Parameter
 from prefect.engine.cache_validators import all_inputs, partial_inputs_only
-from prefect.engine.executors import LocalExecutor
+from prefect.engine.executors import LocalExecutor, DaskExecutor
 from prefect.engine.result import Result
 from prefect.engine.results import LocalResult, PrefectResult
 from prefect.engine.result_handlers import LocalResultHandler, ResultHandler
@@ -2928,8 +2928,12 @@ class TestSaveLoad:
     sys.platform == "win32" or sys.version_info.minor == 6,
     reason="Windows doesn't support any timeout logic",
 )
-@pytest.mark.parametrize("executor", ["local", "sync", "mthread"], indirect=True)
-def test_timeout_actually_stops_execution(executor):
+@pytest.mark.parametrize(
+    "executor", ["local", "sync", "mthread", "mproc_local", "mproc"], indirect=True
+)
+def test_timeout_actually_stops_execution(
+    executor,
+):
     # Note: this is a potentially brittle test! In some cases (local and sync) signal.alarm
     # is used as the mechanism for timing out a task. This passes off the job of measuring
     # the time for the timeout to the OS, which uses the "wallclock" as reference (the real
@@ -2943,15 +2947,25 @@ def test_timeout_actually_stops_execution(executor):
     # the task implementation" we got, but instead do a simple task (create a file) and sleep.
     # This will drastically reduce the brittleness of the test (but not completely).
 
+    # The amount of time to sleep before writing 'invalid' to the file
+    # lower values will decrease test time but increase chances of intermittent failure
+    SLEEP_TIME = 3
+
+    # Determine if the executor is distributed and using daemonic processes which
+    # cannot be cancelled and throw a warning instead.
+    in_daemon_process = isinstance(
+        executor, DaskExecutor
+    ) and not executor.address.startswith("inproc")
+
     with tempfile.TemporaryDirectory() as call_dir:
         # Note: a real file must be used in the case of "mthread"
         FILE = os.path.join(call_dir, "test.txt")
 
-        @prefect.task(timeout=1)
+        @prefect.task(timeout=2)
         def slow_fn():
             with open(FILE, "w") as f:
                 f.write("called!")
-            time.sleep(2)
+            time.sleep(SLEEP_TIME)
             with open(FILE, "a") as f:
                 f.write("invalid")
 
@@ -2962,15 +2976,25 @@ def test_timeout_actually_stops_execution(executor):
         start_time = time.time()
         state = flow.run(executor=executor)
         stop_time = time.time()
-        time.sleep(max(0, 3 - (stop_time - start_time)))
+
+        # Sleep so 'invalid' will be written if the task is not killed, subtracting the
+        # actual runtime to speed up the test a little
+        time.sleep(max(1, SLEEP_TIME - (stop_time - start_time)))
 
         assert os.path.exists(FILE)
         with open(FILE, "r") as f:
-            assert "invalid" not in f.read()
+            # `invalid` should *only be in the file if a daemon process was used
+            assert ("invalid" in f.read()) == in_daemon_process
 
     assert state.is_failed()
     assert isinstance(state.result[slow_fn], TimedOut)
     assert isinstance(state.result[slow_fn].result, TimeoutError)
+    # We cannot capture the UserWarning because it is being run by a Dask worker
+    # but we can make sure the TimeoutError includes a note about it
+    assert (
+        "executed in a daemonic subprocess and will continue to run"
+        in str(state.result[slow_fn].result)
+    ) == in_daemon_process
 
 
 @pytest.mark.skip("Result handlers not yet deprecated")