Repository: PrefectHQ/prefect

Issue:
Flows should not revert to PENDING after one run
Currently, the FlowRunner assigns a PENDING state to a flow that ran but did not complete.

This creates statefulness issues in settings with multiple FlowRunners, because any one FlowRunner doesn't have the ability to declare that the others are no longer RUNNING.

The simple solution(discussed with @cicdw): flow states must progress in just one direction: PENDING -> RUNNING -> FINISHED. "Running" encapsulates any state between "unstarted" and "finished", even if no action is taking place at that moment (such as waiting for a task to retry).

If it ever becomes desirable to truly revert to a PENDING state when no action is taking place, then we could solve the problem like this:
	- each RUNNING state tracks a `run_count`, and setting a new RUNNING state increments that count
	- states can only become PENDING if run_count reaches 0; otherwise they become RUNNING with a decremented run_count
	- We'd probably introduce a new state (maybe subclass RUNNING) for this -- WAITING or something.


Discussion:

Resolving patch:
diff --git a/src/prefect/engine/flow_runner.py b/src/prefect/engine/flow_runner.py
--- a/src/prefect/engine/flow_runner.py
+++ b/src/prefect/engine/flow_runner.py
@@ -447,11 +447,8 @@ def get_flow_run_state(
 
             # check that the flow is finished
             if not all(s.is_finished() for s in terminal_states):
-                self.logger.info("Flow run PENDING: terminal tasks are incomplete.")
-                state = Pending(
-                    message="Some terminal tasks are still pending.",
-                    result=return_states,
-                )
+                self.logger.info("Flow run RUNNING: terminal tasks are incomplete.")
+                state = Running(message="Flow run in progress.", result=return_states)
 
             # check if any key task failed
             elif any(s.is_failed() for s in key_states):


Test patch:
diff --git a/tests/core/test_task_map.py b/tests/core/test_task_map.py
--- a/tests/core/test_task_map.py
+++ b/tests/core/test_task_map.py
@@ -432,7 +432,7 @@ def ll():
         res = div.map(x=ll)
 
     s = f.run(return_tasks=[ll, res], executor=executor)
-    assert s.is_pending()
+    assert s.is_running()
     slist = s.result[res]
     assert slist[0].is_pending()
     assert slist[1].is_successful()
diff --git a/tests/engine/test_flow_runner.py b/tests/engine/test_flow_runner.py
--- a/tests/engine/test_flow_runner.py
+++ b/tests/engine/test_flow_runner.py
@@ -262,12 +262,12 @@ def test_flow_run_method_returns_task_states_even_if_it_doesnt_run():
     flow.add_edge(task1, task2)
 
     flow_state = flow.run(state=Success(), return_tasks=[task1, task2])
-    assert isinstance(flow_state, Success)
-    assert isinstance(flow_state.result[task1], Pending)
-    assert isinstance(flow_state.result[task2], Pending)
+    assert flow_state.is_successful()
+    assert flow_state.result[task1].is_pending()
+    assert flow_state.result[task2].is_pending()
 
 
-def test_flow_runner_remains_pending_if_tasks_are_retrying():
+def test_flow_runner_remains_running_if_tasks_are_retrying():
     # https://github.com/PrefectHQ/prefect/issues/19
     flow = prefect.Flow()
     task1 = SuccessTask()
@@ -276,8 +276,8 @@ def test_flow_runner_remains_pending_if_tasks_are_retrying():
     flow.add_edge(task1, task2)
 
     flow_state = FlowRunner(flow=flow).run(return_tasks=[task1, task2])
-    assert isinstance(flow_state, Pending)
-    assert isinstance(flow_state.result[task1], Success)
+    assert flow_state.is_running()
+    assert flow_state.result[task1].is_successful()
     assert isinstance(flow_state.result[task2], Retrying)
 
 
@@ -350,8 +350,8 @@ def test_flow_run_state_not_determined_by_reference_tasks_if_terminal_tasks_are_
 
     flow.set_reference_tasks([t1])
     flow_state = flow.run(return_tasks=[t1, t2])
-    assert isinstance(flow_state, Pending)
-    assert isinstance(flow_state.result[t1], Failed)
+    assert flow_state.is_running()
+    assert flow_state.result[t1].is_failed()
     assert isinstance(flow_state.result[t2], Retrying)
 
 
@@ -498,7 +498,7 @@ def test_retries_use_cached_inputs(self, executor):
             res = b(a())
 
         first_state = FlowRunner(flow=f).run(executor=executor, return_tasks=[res])
-        assert isinstance(first_state, Pending)
+        assert first_state.is_running()
         b_state = first_state.result[res]
         b_state.cached_inputs = dict(x=2)  # artificially alter state
         with raise_on_exception():  # without caching we'd expect a KeyError
@@ -541,7 +541,7 @@ def test_retries_caches_parameters_as_well(self, executor):
         first_state = FlowRunner(flow=f).run(
             executor=executor, parameters=dict(x=1), return_tasks=[res]
         )
-        assert isinstance(first_state, Pending)
+        assert first_state.is_running()
 
         res_state = first_state.result[res]
         res_state.cached_inputs = dict(x=2)  # artificially alter state
@@ -569,7 +569,7 @@ def test_manual_only_trigger_caches_inputs(self, executor):
         first_state = FlowRunner(flow=f).run(
             executor=executor, parameters=dict(x=11), return_tasks=[res]
         )
-        assert isinstance(first_state, Pending)
+        assert first_state.is_running()
         second_state = FlowRunner(flow=f).run(
             executor=executor,
             parameters=dict(x=1),
@@ -669,7 +669,7 @@ def test_return_failed_includes_retries(self):
             e = ErrorTask(max_retries=1, retry_delay=datetime.timedelta(0))
             s.set_upstream(e)
         state = FlowRunner(flow=f).run(return_failed=True)
-        assert state.is_pending()
+        assert state.is_running()
         assert e in state.result
         assert isinstance(state.result[e], Retrying)
 
@@ -697,7 +697,7 @@ def test_run_count_tracked_via_retry_states(self):
 
         # first run
         state1 = FlowRunner(flow=flow).run(return_tasks=[t1, t2])
-        assert state1.is_pending()
+        assert state1.is_running()
         assert isinstance(state1.result[t1], Retrying)
         assert state1.result[t1].run_count == 1
         assert isinstance(state1.result[t2], Retrying)
@@ -707,7 +707,7 @@ def test_run_count_tracked_via_retry_states(self):
         state2 = FlowRunner(flow=flow).run(
             task_states=state1.result, return_tasks=[t1, t2]
         )
-        assert state2.is_pending()
+        assert state2.is_running()
         assert isinstance(state2.result[t1], Failed)
         assert isinstance(state2.result[t2], Retrying)
         assert state2.result[t2].run_count == 2