Repository: apache/airflow

Issue:
Don't include upstream_failed in the 'active task instances' test in the scheduler
**Apache Airflow version**: 2.0.0a1

**What happened**:

On my dev environment I have a lot of DAG runs with failed tasks, where the *downstream* task instances then have been set to `upstream_failed`. The scheduler is now refusing to schedule more instances for such DAGs.

The scheduler looks for such task instances that are not in the `State.finished` set:

https://github.com/apache/airflow/blob/db3fe0926bb75008311eed804052c90bfa912424/airflow/jobs/scheduler_job.py#L1482-L1488

However, `State.finished` is currently not a proper inverse of *running* states, it only includes success, failed, and skipped:

https://github.com/apache/airflow/blob/db3fe0926bb75008311eed804052c90bfa912424/airflow/utils/state.py#L110-L114

What is missing is the task instance `upstream_failed` state, so currently my log is being filled (very rapidly) with:

```
[2020-10-18 13:22:18,503] {{scheduler_job.py:1658}} INFO - DAG test_dag already has 26 active runs, not queuing any more tasks
```

Those 26 'active runs' are simply dag runs that failed and have 'upstream_failed' task instances.

Either `upstream_failed` should be added to `State.finished`, or the Scheduler task instance test should explicitly take that state into account.


Discussion:
Yup we will fix this for alpha2. Thanks for reporting
I've been running with the following patch for a few days now (added to `airflow_local_config.py` to ensure early loading):

```
def airflow_issue_11633_upstream_failed_state():
    """The 'upstream_failed' state should be included in the 'finished' set"""
    from airflow.utils.state import State

    # in Airflow =<2.0.0a1, State.finished is a method, in future versions it is a frozenset.
    finished, is_set = State.finished, isinstance(State.finished, frozenset)
    if not is_set:
        finished = finished()

    if State.UPSTREAM_FAILED in finished:
        return

    if is_set:
        State.finished = frozenset([State.UPSTREAM_FAILED]) | State.finished
    else:

        @classmethod
        def finished_with_failed(cls):
            return [*finished, State.UPSTREAM_FAILED]

        State.finished = finished_with_failed


airflow_issue_11633_upstream_failed_state()
```

and I have not seen any adverse side effects so far.
I'm re-working that query in question a bit as part of fixing #11582, I'll encorporate this fix in there and add a test case for it

Resolving patch:
diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py
--- a/airflow/models/dagrun.py
+++ b/airflow/models/dagrun.py
@@ -387,7 +387,7 @@ def update_state(
             ti.task = dag.get_task(ti.task_id)
 
         unfinished_tasks = [t for t in tis if t.state in State.unfinished]
-        finished_tasks = [t for t in tis if t.state in State.finished | {State.UPSTREAM_FAILED}]
+        finished_tasks = [t for t in tis if t.state in State.finished]
         none_depends_on_past = all(not t.task.depends_on_past for t in unfinished_tasks)
         none_task_concurrency = all(t.task.task_concurrency is None for t in unfinished_tasks)
         if unfinished_tasks:
diff --git a/airflow/ti_deps/dep_context.py b/airflow/ti_deps/dep_context.py
--- a/airflow/ti_deps/dep_context.py
+++ b/airflow/ti_deps/dep_context.py
@@ -100,7 +100,7 @@ def ensure_finished_tasks(self, dag, execution_date: pendulum.DateTime, session:
             self.finished_tasks = dag.get_task_instances(
                 start_date=execution_date,
                 end_date=execution_date,
-                state=State.finished | {State.UPSTREAM_FAILED},
+                state=State.finished,
                 session=session,
             )
         return self.finished_tasks
diff --git a/airflow/utils/state.py b/airflow/utils/state.py
--- a/airflow/utils/state.py
+++ b/airflow/utils/state.py
@@ -107,11 +107,15 @@ def color_fg(cls, state):
         SUCCESS,
         FAILED,
         SKIPPED,
+        UPSTREAM_FAILED,
     ])
     """
-    A list of states indicating that a task started and completed a
-    run attempt. Note that the attempt could have resulted in failure or
-    have been interrupted; in any case, it is no longer running.
+    A list of states indicating a task has reached a terminal state (i.e. it has "finished") and needs no
+    further action.
+
+    Note that the attempt could have resulted in failure or have been
+    interrupted; or perhaps never run at all (skip, or upstream_failed) in any
+    case, it is no longer running.
     """
 
     unfinished = frozenset([


Test patch:
diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py
--- a/tests/jobs/test_scheduler_job.py
+++ b/tests/jobs/test_scheduler_job.py
@@ -50,6 +50,7 @@
 from airflow.serialization.serialized_objects import SerializedDAG
 from airflow.utils import timezone
 from airflow.utils.callback_requests import DagCallbackRequest, TaskCallbackRequest
+from airflow.utils.dag_processing import DagFileProcessorAgent
 from airflow.utils.dates import days_ago
 from airflow.utils.file import list_py_file_paths
 from airflow.utils.session import create_session, provide_session
@@ -3514,6 +3515,64 @@ def test_scheduler_sets_job_id_on_dag_run(self):
 
         assert dag.get_last_dagrun().creating_job_id == scheduler.id
 
+    def test_do_schedule_max_active_runs_upstream_failed(self):
+        """
+        Test that tasks in upstream failed don't count as actively running.
+
+        This test can be removed when adding a queued state to DagRuns.
+        """
+
+        with DAG(
+            dag_id='test_max_active_run_plus_manual_trigger',
+            start_date=DEFAULT_DATE,
+            schedule_interval='@once',
+            max_active_runs=1,
+        ) as dag:
+            # Cant use DummyOperator as that goes straight to success
+            task1 = BashOperator(task_id='dummy1', bash_command='true')
+
+        session = settings.Session()
+        dagbag = DagBag(
+            dag_folder=os.devnull,
+            include_examples=False,
+            read_dags_from_db=True,
+        )
+
+        dagbag.bag_dag(dag=dag, root_dag=dag)
+        dagbag.sync_to_db(session=session)
+
+        run1 = dag.create_dagrun(
+            run_type=DagRunType.SCHEDULED,
+            execution_date=DEFAULT_DATE,
+            state=State.RUNNING,
+            session=session,
+        )
+
+        ti = run1.get_task_instance(task1.task_id, session)
+        ti.state = State.UPSTREAM_FAILED
+
+        run2 = dag.create_dagrun(
+            run_type=DagRunType.SCHEDULED,
+            execution_date=DEFAULT_DATE + timedelta(hours=1),
+            state=State.RUNNING,
+            session=session,
+        )
+
+        dag.sync_to_db(session=session)  # Update the date fields
+
+        job = SchedulerJob()
+        job.executor = MockExecutor(do_update=False)
+        job.processor_agent = mock.MagicMock(spec=DagFileProcessorAgent)
+
+        num_queued = job._do_scheduling(session)
+        session.flush()
+
+        assert num_queued == 1
+        ti = run2.get_task_instance(task1.task_id, session)
+        assert ti.state == State.QUEUED
+
+        session.rollback()
+
 
 @pytest.mark.xfail(reason="Work out where this goes")
 def test_task_with_upstream_skip_process_task_instances():
diff --git a/tests/ti_deps/deps/test_trigger_rule_dep.py b/tests/ti_deps/deps/test_trigger_rule_dep.py
--- a/tests/ti_deps/deps/test_trigger_rule_dep.py
+++ b/tests/ti_deps/deps/test_trigger_rule_dep.py
@@ -547,8 +547,7 @@ def test_get_states_count_upstream_ti(self):
         finished_tasks = DepContext().ensure_finished_tasks(ti_op2.task.dag, ti_op2.execution_date, session)
         self.assertEqual(get_states_count_upstream_ti(finished_tasks=finished_tasks, ti=ti_op2),
                          (1, 0, 0, 0, 1))
-        finished_tasks = dr.get_task_instances(state=State.finished | {State.UPSTREAM_FAILED},
-                                               session=session)
+        finished_tasks = dr.get_task_instances(state=State.finished, session=session)
         self.assertEqual(get_states_count_upstream_ti(finished_tasks=finished_tasks, ti=ti_op4),
                          (1, 0, 1, 0, 2))
         self.assertEqual(get_states_count_upstream_ti(finished_tasks=finished_tasks, ti=ti_op5),