Repository: PrefectHQ/prefect

Issue:
signals.TRIGGERFAIL leading to retry attemp reset
Each TriggerFailed state should increment retry attempt number and if retry runs out the task is deemed as a Failed state and the next task starts to run.
However, when I executed following code.
```
import datetime
from prefect import Flow, Task
from prefect.triggers import manual_only
from prefect.engine import state, signals


class TaskFactory(Task):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def run(self):
        print('Task starts running ------------------------------------------------- Task starts running')
        raise signals.TRIGGERFAIL('Failed due to no external triggering from host')


def on_state_change(task, old_state, new_state):
    print('current work flow running state is '+str(new_state))
    if isinstance(new_state, state.Paused):
        return state.Resume()
    if isinstance(new_state, state.TriggerFailed):
        print('No triggering from host.')


flow = Flow('demo_flow')
tasks_in_sequence = []
for _ in range(3):
    current_task = TaskFactory(name='demo_task_'+str(_), max_retries=5, retry_delay=datetime.timedelta(seconds=1),
                               trigger=manual_only, state_handlers=[on_state_change], timeout=2,
                               skip_on_upstream_skip=False)
    flow.add_task(current_task)
    tasks_in_sequence.append(current_task)

flow.chain(*tasks_in_sequence)

flow.run()
```
I was always stuck at the first task.
```
[2019-06-25 02:36:17,690] INFO - prefect.FlowRunner | Beginning Flow run for 'demo_flow'
current work flow running state is <Paused: "Trigger function is "manual_only"">
[2019-06-25 02:36:17,690] INFO - prefect.FlowRunner | Starting flow run.
[2019-06-25 02:36:17,690] INFO - prefect.TaskRunner | Task 'demo_task_0': Starting task run...
[2019-06-25 02:36:17,690] INFO - prefect.TaskRunner | Task 'demo_task_0': finished task run for task with final state: 'Resume'
[2019-06-25 02:36:17,690] INFO - prefect.TaskRunner | Task 'demo_task_1': Starting task run...
[2019-06-25 02:36:17,690] INFO - prefect.TaskRunner | Task 'demo_task_1': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:17,690] INFO - prefect.TaskRunner | Task 'demo_task_2': Starting task run...
[2019-06-25 02:36:17,690] INFO - prefect.TaskRunner | Task 'demo_task_2': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:17,690] INFO - prefect.FlowRunner | Flow run RUNNING: terminal tasks are incomplete.
[2019-06-25 02:36:17,690] INFO - prefect.FlowRunner | Beginning Flow run for 'demo_flow'
current work flow running state is <Running: "Starting task run.">
Task starts running ------------------------------------------------- Task starts running
current work flow running state is <TriggerFailed: "Failed due to no external triggering from host">
No triggering from host.
current work flow running state is <Retrying: "Retrying Task (after attempt 1 of 6)">
[2019-06-25 02:36:17,706] INFO - prefect.TaskRunner | Task 'demo_task_0': Starting task run...
[2019-06-25 02:36:17,706] INFO - prefect.TaskRunner | Task 'demo_task_0': finished task run for task with final state: 'Retrying'
[2019-06-25 02:36:17,706] INFO - prefect.TaskRunner | Task 'demo_task_1': Starting task run...
[2019-06-25 02:36:17,706] INFO - prefect.TaskRunner | Task 'demo_task_1': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:17,706] INFO - prefect.TaskRunner | Task 'demo_task_2': Starting task run...
[2019-06-25 02:36:17,706] INFO - prefect.TaskRunner | Task 'demo_task_2': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:17,706] INFO - prefect.FlowRunner | Flow run RUNNING: terminal tasks are incomplete.
[2019-06-25 02:36:17,706] INFO - prefect.Flow | Waiting for next available Task run at 2019-06-25T02:36:18.706132+00:00
[2019-06-25 02:36:18,721] INFO - prefect.FlowRunner | Beginning Flow run for 'demo_flow'
[2019-06-25 02:36:18,721] INFO - prefect.TaskRunner | Task 'demo_task_0': Starting task run...
current work flow running state is <Paused: "Trigger function is "manual_only"">
current work flow running state is <Running: "Starting task run.">
Task starts running ------------------------------------------------- Task starts running
current work flow running state is <TriggerFailed: "Failed due to no external triggering from host">
No triggering from host.
current work flow running state is <Retrying: "Retrying Task (after attempt 1 of 6)">
[2019-06-25 02:36:18,721] INFO - prefect.TaskRunner | Task 'demo_task_0': finished task run for task with final state: 'Resume'
[2019-06-25 02:36:18,721] INFO - prefect.TaskRunner | Task 'demo_task_1': Starting task run...
[2019-06-25 02:36:18,721] INFO - prefect.TaskRunner | Task 'demo_task_1': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:18,721] INFO - prefect.TaskRunner | Task 'demo_task_2': Starting task run...
[2019-06-25 02:36:18,721] INFO - prefect.TaskRunner | Task 'demo_task_2': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:18,721] INFO - prefect.FlowRunner | Flow run RUNNING: terminal tasks are incomplete.
[2019-06-25 02:36:18,721] INFO - prefect.FlowRunner | Beginning Flow run for 'demo_flow'
[2019-06-25 02:36:18,721] INFO - prefect.TaskRunner | Task 'demo_task_0': Starting task run...
[2019-06-25 02:36:18,721] INFO - prefect.TaskRunner | Task 'demo_task_0': finished task run for task with final state: 'Retrying'
[2019-06-25 02:36:18,721] INFO - prefect.TaskRunner | Task 'demo_task_1': Starting task run...
[2019-06-25 02:36:18,721] INFO - prefect.TaskRunner | Task 'demo_task_1': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:18,721] INFO - prefect.TaskRunner | Task 'demo_task_2': Starting task run...
[2019-06-25 02:36:18,721] INFO - prefect.TaskRunner | Task 'demo_task_2': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:18,721] INFO - prefect.FlowRunner | Flow run RUNNING: terminal tasks are incomplete.
[2019-06-25 02:36:18,721] INFO - prefect.Flow | Waiting for next available Task run at 2019-06-25T02:36:19.721694+00:00
current work flow running state is <Paused: "Trigger function is "manual_only"">
[2019-06-25 02:36:19,737] INFO - prefect.FlowRunner | Beginning Flow run for 'demo_flow'
current work flow running state is <Running: "Starting task run.">
[2019-06-25 02:36:19,737] INFO - prefect.TaskRunner | Task 'demo_task_0': Starting task run...
[2019-06-25 02:36:19,737] INFO - prefect.TaskRunner | Task 'demo_task_0': finished task run for task with final state: 'Resume'
[2019-06-25 02:36:19,737] INFO - prefect.TaskRunner | Task 'demo_task_1': Starting task run...
Task starts running ------------------------------------------------- Task starts running
[2019-06-25 02:36:19,737] INFO - prefect.TaskRunner | Task 'demo_task_1': finished task run for task with final state: 'Pending'
current work flow running state is <TriggerFailed: "Failed due to no external triggering from host">
[2019-06-25 02:36:19,737] INFO - prefect.TaskRunner | Task 'demo_task_2': Starting task run...
No triggering from host.
current work flow running state is <Retrying: "Retrying Task (after attempt 1 of 6)">
[2019-06-25 02:36:19,737] INFO - prefect.TaskRunner | Task 'demo_task_2': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:19,737] INFO - prefect.FlowRunner | Flow run RUNNING: terminal tasks are incomplete.
[2019-06-25 02:36:19,737] INFO - prefect.FlowRunner | Beginning Flow run for 'demo_flow'
[2019-06-25 02:36:19,737] INFO - prefect.TaskRunner | Task 'demo_task_0': Starting task run...
[2019-06-25 02:36:19,737] INFO - prefect.TaskRunner | Task 'demo_task_0': finished task run for task with final state: 'Retrying'
[2019-06-25 02:36:19,737] INFO - prefect.TaskRunner | Task 'demo_task_1': Starting task run...
[2019-06-25 02:36:19,737] INFO - prefect.TaskRunner | Task 'demo_task_1': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:19,737] INFO - prefect.TaskRunner | Task 'demo_task_2': Starting task run...
[2019-06-25 02:36:19,737] INFO - prefect.TaskRunner | Task 'demo_task_2': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:19,737] INFO - prefect.FlowRunner | Flow run RUNNING: terminal tasks are incomplete.
[2019-06-25 02:36:19,737] INFO - prefect.Flow | Waiting for next available Task run at 2019-06-25T02:36:20.737258+00:00
current work flow running state is <Paused: "Trigger function is "manual_only"">
[2019-06-25 02:36:20,752] INFO - prefect.FlowRunner | Beginning Flow run for 'demo_flow'
[2019-06-25 02:36:20,752] INFO - prefect.TaskRunner | Task 'demo_task_0': Starting task run...
current work flow running state is <Running: "Starting task run.">
Task starts running ------------------------------------------------- Task starts running
[2019-06-25 02:36:20,752] INFO - prefect.TaskRunner | Task 'demo_task_0': finished task run for task with final state: 'Resume'
current work flow running state is <TriggerFailed: "Failed due to no external triggering from host">
[2019-06-25 02:36:20,752] INFO - prefect.TaskRunner | Task 'demo_task_1': Starting task run...
No triggering from host.
[2019-06-25 02:36:20,752] INFO - prefect.TaskRunner | Task 'demo_task_1': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:20,752] INFO - prefect.TaskRunner | Task 'demo_task_2': Starting task run...
[2019-06-25 02:36:20,752] INFO - prefect.TaskRunner | Task 'demo_task_2': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:20,752] INFO - prefect.FlowRunner | Flow run RUNNING: terminal tasks are incomplete.
[2019-06-25 02:36:20,752] INFO - prefect.FlowRunner | Beginning Flow run for 'demo_flow'
current work flow running state is <Retrying: "Retrying Task (after attempt 1 of 6)">
[2019-06-25 02:36:20,752] INFO - prefect.TaskRunner | Task 'demo_task_0': Starting task run...
[2019-06-25 02:36:20,752] INFO - prefect.TaskRunner | Task 'demo_task_0': finished task run for task with final state: 'Retrying'
[2019-06-25 02:36:20,752] INFO - prefect.TaskRunner | Task 'demo_task_1': Starting task run...
[2019-06-25 02:36:20,752] INFO - prefect.TaskRunner | Task 'demo_task_1': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:20,752] INFO - prefect.TaskRunner | Task 'demo_task_2': Starting task run...
[2019-06-25 02:36:20,752] INFO - prefect.TaskRunner | Task 'demo_task_2': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:20,752] INFO - prefect.FlowRunner | Flow run RUNNING: terminal tasks are incomplete.
[2019-06-25 02:36:20,752] INFO - prefect.Flow | Waiting for next available Task run at 2019-06-25T02:36:21.752814+00:00
current work flow running state is <Paused: "Trigger function is "manual_only"">
current work flow running state is <Running: "Starting task run.">
[2019-06-25 02:36:21,768] INFO - prefect.FlowRunner | Beginning Flow run for 'demo_flow'
Task starts running ------------------------------------------------- Task starts running
current work flow running state is <TriggerFailed: "Failed due to no external triggering from host">
No triggering from host.
[2019-06-25 02:36:21,768] INFO - prefect.TaskRunner | Task 'demo_task_0': Starting task run...
current work flow running state is <Retrying: "Retrying Task (after attempt 1 of 6)">
[2019-06-25 02:36:21,768] INFO - prefect.TaskRunner | Task 'demo_task_0': finished task run for task with final state: 'Resume'
[2019-06-25 02:36:21,768] INFO - prefect.TaskRunner | Task 'demo_task_1': Starting task run...
[2019-06-25 02:36:21,768] INFO - prefect.TaskRunner | Task 'demo_task_1': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:21,768] INFO - prefect.TaskRunner | Task 'demo_task_2': Starting task run...
[2019-06-25 02:36:21,768] INFO - prefect.TaskRunner | Task 'demo_task_2': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:21,768] INFO - prefect.FlowRunner | Flow run RUNNING: terminal tasks are incomplete.
[2019-06-25 02:36:21,768] INFO - prefect.FlowRunner | Beginning Flow run for 'demo_flow'
[2019-06-25 02:36:21,768] INFO - prefect.TaskRunner | Task 'demo_task_0': Starting task run...
[2019-06-25 02:36:21,768] INFO - prefect.TaskRunner | Task 'demo_task_0': finished task run for task with final state: 'Retrying'
[2019-06-25 02:36:21,768] INFO - prefect.TaskRunner | Task 'demo_task_1': Starting task run...
[2019-06-25 02:36:21,768] INFO - prefect.TaskRunner | Task 'demo_task_1': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:21,768] INFO - prefect.TaskRunner | Task 'demo_task_2': Starting task run...
[2019-06-25 02:36:21,768] INFO - prefect.TaskRunner | Task 'demo_task_2': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:21,768] INFO - prefect.FlowRunner | Flow run RUNNING: terminal tasks are incomplete.
[2019-06-25 02:36:21,768] INFO - prefect.Flow | Waiting for next available Task run at 2019-06-25T02:36:22.768355+00:00
[2019-06-25 02:36:22,783] INFO - prefect.FlowRunner | Beginning Flow run for 'demo_flow'
current work flow running state is <Paused: "Trigger function is "manual_only"">
[2019-06-25 02:36:22,783] INFO - prefect.TaskRunner | Task 'demo_task_0': Starting task run...
current work flow running state is <Running: "Starting task run.">
[2019-06-25 02:36:22,783] INFO - prefect.TaskRunner | Task 'demo_task_0': finished task run for task with final state: 'Resume'
[2019-06-25 02:36:22,783] INFO - prefect.TaskRunner | Task 'demo_task_1': Starting task run...
[2019-06-25 02:36:22,783] INFO - prefect.TaskRunner | Task 'demo_task_1': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:22,783] INFO - prefect.TaskRunner | Task 'demo_task_2': Starting task run...
Task starts running ------------------------------------------------- Task starts running
current work flow running state is <TriggerFailed: "Failed due to no external triggering from host">
No triggering from host.
current work flow running state is <Retrying: "Retrying Task (after attempt 1 of 6)">
[2019-06-25 02:36:22,783] INFO - prefect.TaskRunner | Task 'demo_task_2': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:22,783] INFO - prefect.FlowRunner | Flow run RUNNING: terminal tasks are incomplete.
[2019-06-25 02:36:22,783] INFO - prefect.FlowRunner | Beginning Flow run for 'demo_flow'
[2019-06-25 02:36:22,783] INFO - prefect.TaskRunner | Task 'demo_task_0': Starting task run...
[2019-06-25 02:36:22,783] INFO - prefect.TaskRunner | Task 'demo_task_0': finished task run for task with final state: 'Retrying'
[2019-06-25 02:36:22,783] INFO - prefect.TaskRunner | Task 'demo_task_1': Starting task run...
[2019-06-25 02:36:22,783] INFO - prefect.TaskRunner | Task 'demo_task_1': finished task run for task with final state: 'Pending'
[2019-06-25 02:36:22,783] INFO - prefect.TaskRunner | Task 'demo_task_2': Starting task run...
```




Discussion:
After some poking, I found the task's trigger argument set to "manual_only" might be the root cause that leads to resetting the retry attempt counting each time it's invoked.
Hi @occoder - I'm having a hard time understanding what you expect the behavior to be here?  
Hi @cicdw The failed state leads that the flow runner auto transit to retrying state which in turn should increment the attemptted retry counter by one. However, the example shown above did not increment the retry counter meaning the task kept retrying and never reached the max retry limit.

Resolving patch:
diff --git a/src/prefect/engine/state.py b/src/prefect/engine/state.py
--- a/src/prefect/engine/state.py
+++ b/src/prefect/engine/state.py
@@ -280,6 +280,9 @@ def __init__(
     ):
         super().__init__(message=message, result=result, cached_inputs=cached_inputs)
         self.start_time = pendulum.instance(start_time or pendulum.now("utc"))
+        run_count = prefect.context.get("task_run_count")
+        if run_count is not None:
+            self.context.update(task_run_count=run_count)
 
 
 class Paused(Scheduled):
diff --git a/src/prefect/engine/task_runner.py b/src/prefect/engine/task_runner.py
--- a/src/prefect/engine/task_runner.py
+++ b/src/prefect/engine/task_runner.py
@@ -144,7 +144,7 @@ def initialize_run(  # type: ignore
         if isinstance(state, Retrying):
             run_count = state.run_count + 1
         else:
-            run_count = 1
+            run_count = state.context.get("task_run_count", 1)
 
         if isinstance(state, Resume):
             context.update(resume=True)


Test patch:
diff --git a/tests/core/test_flow.py b/tests/core/test_flow.py
--- a/tests/core/test_flow.py
+++ b/tests/core/test_flow.py
@@ -17,11 +17,13 @@
 from prefect.core.task import Parameter, Task
 from prefect.engine.cache_validators import all_inputs, partial_inputs_only
 from prefect.engine.result_handlers import LocalResultHandler, ResultHandler
-from prefect.engine.signals import LOOP, PrefectError
+from prefect.engine.signals import PrefectError, FAIL, LOOP
 from prefect.engine.state import (
     Failed,
     Finished,
     Mapped,
+    Paused,
+    Resume,
     Pending,
     Skipped,
     State,
@@ -2206,6 +2208,31 @@ def downstream(l):
     assert flow_state.result[final].result == 200 ** 2
 
 
+def test_pause_resume_works_with_retries():
+    runs = []
+
+    def state_handler(obj, old, new):
+        if isinstance(new, Paused):
+            return Resume()
+        elif old.is_running():
+            raise FAIL("cant pass go")
+
+    @task(
+        max_retries=2,
+        retry_delay=datetime.timedelta(seconds=0),
+        state_handlers=[state_handler],
+        trigger=prefect.triggers.manual_only,
+    )
+    def fail():
+        runs.append(1)
+
+    f = Flow("huh", tasks=[fail])
+    flow_state = f.run()
+
+    assert flow_state.is_failed()
+    assert len(runs) == 3
+
+
 def test_looping_with_retries_works_in_a_flow():
     @task(max_retries=1, retry_delay=datetime.timedelta(seconds=0))
     def looper(x):
diff --git a/tests/engine/test_state.py b/tests/engine/test_state.py
--- a/tests/engine/test_state.py
+++ b/tests/engine/test_state.py
@@ -139,6 +139,25 @@ def test_only_scheduled_and_queued_states_have_start_times(cls):
         assert not state.is_scheduled()
 
 
+@pytest.mark.parametrize("cls", all_states)
+def test_only_scheduled_states_have_task_run_count_in_context(cls):
+    """
+    Storing task_run_count in state.context provides a way of communicating
+    the current run_count across multiple state changes.  Persisting this data
+    in a Finished state causes the run count to "freeze" across runs.
+    """
+    with prefect.context(task_run_count=910):
+        state = cls()
+
+    if state.context.get("task_run_count") is not None:
+        assert isinstance(state, Scheduled)
+        assert state.is_scheduled()
+        assert state.context["task_run_count"] == 910
+    else:
+        assert not isinstance(state, Scheduled)
+        assert not state.is_scheduled()
+
+
 def test_retry_stores_loop_count():
     state = Looped(loop_count=2)
     assert state.loop_count == 2
diff --git a/tests/serialization/test_states.py b/tests/serialization/test_states.py
--- a/tests/serialization/test_states.py
+++ b/tests/serialization/test_states.py
@@ -152,6 +152,22 @@ def test_serialize_state_with_context():
     assert deserialized.context == dict(tags=["foo", "bar"])
 
 
+def test_serialize_scheduled_state_with_context():
+    with prefect.context(task_run_count=42):
+        s = state.Scheduled(message="hi")
+
+    serialized = StateSchema().dump(s)
+    assert isinstance(serialized, dict)
+    assert serialized["type"] == "Scheduled"
+    assert serialized["message"] == "hi"
+    assert serialized["__version__"] == prefect.__version__
+    assert serialized["context"] == dict(task_run_count=42)
+
+    deserialized = StateSchema().load(serialized)
+    assert deserialized.is_scheduled()
+    assert deserialized.context == dict(task_run_count=42)
+
+
 def test_serialize_state_with_context_allows_for_diverse_values():
     s = state.Running(message="hi")
     s.context = dict(tags=["foo", "bar"], info=dict(x=42), baz="99")