# Buggy code
def _executable_task_instances_to_queued(
    self, max_tis: int, session: Session = None
) -> List[TI]:
    """
    Finds TIs that are ready for execution with respect to pool limits,
    dag concurrency, executor state, and priority.

    :param max_tis: Maximum number of TIs to queue in this loop.
    :type max_tis: int
    :return: list[airflow.models.TaskInstance]
    """
    executable_tis: List[TI] = []

    # Get the pool settings. We get a lock on the pool rows, treating this as a "critical section"
    # Throws an exception if lock cannot be obtained, rather than blocking
    pools = models.Pool.slots_stats(lock_rows=True, session=session)

    # If the pools are full, there is no point doing anything!
    # If _somehow_ the pool is overfull, don't let the limit go negative - it breaks SQL
    pool_slots_free = max(0, sum(pool["open"] for pool in pools.values()))

    if pool_slots_free == 0:
        self.log.debug("All pools are full!")
        return executable_tis

    max_tis = min(max_tis, pool_slots_free)

    # Get all task instances associated with scheduled
    # DagRuns which are not backfilled, in the given states,
    # and the dag is not paused
    query = (
        session.query(TI)
        .outerjoin(TI.dag_run)
        .filter(or_(DR.run_id.is_(None), DR.run_type != DagRunType.BACKFILL_JOB))
        .join(TI.dag_model)
        .filter(not_(DM.is_paused))
        .filter(TI.state == State.SCHEDULED)
        .options(selectinload("dag_model"))
        .limit(max_tis)
    )

    task_instances_to_examine: List[TI] = with_row_locks(
        query,
        of=TI,
        **skip_locked(session=session),
    ).all()
    # TODO[HA]: This was wrong before anyway, as it only looked at a sub-set of dags, not everything.
    # Stats.gauge('scheduler.tasks.pending', len(task_instances_to_examine))

    if len(task_instances_to_examine) == 0:
        self.log.debug("No tasks to consider for execution.")
        return executable_tis

    # Put one task instance on each line
    task_instance_str = "\n\t".join([repr(x) for x in task_instances_to_examine])
    self.log.info(
        "%s tasks up for execution:\n\t%s",
        len(task_instances_to_examine),
        task_instance_str,
    )

    pool_to_task_instances: DefaultDict[str, List[models.Pool]] = defaultdict(list)
    for task_instance in task_instances_to_examine:
        pool_to_task_instances[task_instance.pool].append(task_instance)

    # dag_id to # of running tasks and (dag_id, task_id) to # of running tasks.
    dag_concurrency_map: DefaultDict[str, int]
    task_concurrency_map: DefaultDict[Tuple[str, str], int]
    dag_concurrency_map, task_concurrency_map = self.__get_concurrency_maps(
        states=list(EXECUTION_STATES), session=session
    )

    num_tasks_in_executor = 0
    # Number of tasks that cannot be scheduled because of no open slot in pool
    num_starving_tasks_total = 0

    # Go through each pool, and queue up a task for execution if there are
    # any open slots in the pool.
    # pylint: disable=too-many-nested-blocks
    for pool, task_instances in pool_to_task_instances.items():
        pool_name = pool
        if pool not in pools:
            self.log.warning(
                "Tasks using non-existent pool '%s' will not be scheduled", pool
            )
            continue

        open_slots = pools[pool]["open"]

        num_ready = len(task_instances)
        self.log.info(
            "Figuring out tasks to run in Pool(name=%s) with %s open slots "
            "and %s task instances ready to be queued",
            pool,
            open_slots,
            num_ready,
        )

        priority_sorted_task_instances = sorted(
            task_instances, key=lambda ti: (-ti.priority_weight, ti.execution_date)
        )

        num_starving_tasks = 0
        for current_index, task_instance in enumerate(priority_sorted_task_instances):
            if open_slots <= 0:
                self.log.info(
                    "Not scheduling since there are %s open slots in pool %s",
                    open_slots,
                    pool,
                )
                # Can't schedule any more since there are no more open slots.
                num_unhandled = len(priority_sorted_task_instances) - current_index
                num_starving_tasks += num_unhandled
                num_starving_tasks_total += num_unhandled
                break

            # Check to make sure that the task concurrency of the DAG hasn't been
            # reached.
            dag_id = task_instance.dag_id

            current_dag_concurrency = dag_concurrency_map[dag_id]
            dag_concurrency_limit = task_instance.dag_model.concurrency
            self.log.info(
                "DAG %s has %s/%s running and queued tasks",
                dag_id,
                current_dag_concurrency,
                dag_concurrency_limit,
            )
            if current_dag_concurrency >= dag_concurrency_limit:
                self.log.info(
                    "Not executing %s since the number of tasks running or queued "
                    "from DAG %s is >= to the DAG's task concurrency limit of %s",
                    task_instance,
                    dag_id,
                    dag_concurrency_limit,
                )
                continue

            task_concurrency_limit: Optional[int] = None
            if task_instance.dag_model.has_task_concurrency_limits:
                # Many dags don't have a task_concurrency, so where we can avoid loading the full
                # serialized DAG the better.
                serialized_dag = self.dagbag.get_dag(dag_id, session=session)
                if serialized_dag.has_task(task_instance.task_id):
                    task_concurrency_limit = serialized_dag.get_task(
                        task_instance.task_id
                    ).task_concurrency

                if task_concurrency_limit is not None:
                    current_task_concurrency = task_concurrency_map[
                        (task_instance.dag_id, task_instance.task_id)
                    ]

                    if current_task_concurrency >= task_concurrency_limit:
                        self.log.info(
                            "Not executing %s since the task concurrency for"
                            " this task has been reached.",
                            task_instance,
                        )
                        continue

            if task_instance.pool_slots > open_slots:
                self.log.info(
                    "Not executing %s since it requires %s slots "
                    "but there are %s open slots in the pool %s.",
                    task_instance,
                    task_instance.pool_slots,
                    open_slots,
                    pool,
                )
                num_starving_tasks += 1
                num_starving_tasks_total += 1
                # Though we can execute tasks with lower priority if there's enough room
                continue

            executable_tis.append(task_instance)
            open_slots -= task_instance.pool_slots
            dag_concurrency_map[dag_id] += 1
            task_concurrency_map[(task_instance.dag_id, task_instance.task_id)] += 1

        Stats.gauge(f"pool.starving_tasks.{pool_name}", num_starving_tasks)

    Stats.gauge("scheduler.tasks.starving", num_starving_tasks_total)
    Stats.gauge("scheduler.tasks.running", num_tasks_in_executor)
    Stats.gauge("scheduler.tasks.executable", len(executable_tis))

    task_instance_str = "\n\t".join([repr(x) for x in executable_tis])
    self.log.info(
        "Setting the following tasks to queued state:\n\t%s", task_instance_str
    )

    # set TIs to queued state
    filter_for_tis = TI.filter_for_tis(executable_tis)
    session.query(TI).filter(filter_for_tis).update(
        # TODO[ha]: should we use func.now()? How does that work with DB timezone on mysql when it's not
        # UTC?
        {
            TI.state: State.QUEUED,
            TI.queued_dttm: timezone.utcnow(),
            TI.queued_by_job_id: self.id,
        },
        synchronize_session=False,
    )

    for ti in executable_tis:
        make_transient(ti)
    return executable_tis


# Traceback (_mysql_exceptions.OperationalError)
[2020-10-26 09:03:54,608] {{settings.py:49}} INFO - Configured default timezone Timezone('UTC')
[2020-10-26 09:04:05,467] {{scheduler_job.py:1327}} ERROR - Exception when executing SchedulerJob._run_scheduler_loop
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context
self.dialect.do_execute(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 255, in execute
self.errorhandler(self, exc, value)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/connections.py", line 50, in defaulterrorhandler
raise errorvalue
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 252, in execute
res = self._query(query)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 379, in _query
self._do_get_result(db)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 182, in _do_get_result
self._result = result = self._get_result()
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 411, in _get_result
return self._get_db().store_result()
_mysql_exceptions.OperationalError: (1213, 'Deadlock found when trying to get lock; try restarting transaction')

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py", line 1308, in _execute
self._run_scheduler_loop()
File "/home/airflow/.local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py", line 1379, in _run_scheduler_loop
num_queued_tis = self._do_scheduling(session)
File "/home/airflow/.local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py", line 1451, in _do_scheduling
self._create_dag_runs(query.all(), session)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3341, in all
return list(self)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3503, in __iter__
return self._execute_and_instances(context)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3528, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1014, in execute
return meth(self, multiparams, params)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1127, in _execute_clauseelement
ret = self._execute_context(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1317, in _execute_context
self._handle_dbapi_exception(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1511, in _handle_dbapi_exception
util.raise_(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
raise exception
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context
self.dialect.do_execute(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 255, in execute
self.errorhandler(self, exc, value)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/connections.py", line 50, in defaulterrorhandler
raise errorvalue
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 252, in execute
res = self._query(query)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 379, in _query
self._do_get_result(db)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 182, in _do_get_result
self._result = result = self._get_result()
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 411, in _get_result
return self._get_db().store_result()
sqlalchemy.exc.OperationalError: (_mysql_exceptions.OperationalError) (1213, 'Deadlock found when trying to get lock; try restarting transaction')
[SQL: SELECT dag.dag_id AS dag_dag_id, dag.root_dag_id AS dag_root_dag_id, dag.is_paused AS dag_is_paused, dag.is_subdag AS dag_is_subdag, dag.is_active AS dag_is_active, dag.last_scheduler_run AS dag_last_scheduler_run, dag.last_pickled AS dag_last_pickled, dag.last_expired AS dag_last_expired, dag.scheduler_lock AS dag_scheduler_lock, dag.pickle_id AS dag_pickle_id, dag.fileloc AS dag_fileloc, dag.owners AS dag_owners, dag.description AS dag_description, dag.default_view AS dag_default_view, dag.schedule_interval AS dag_schedule_interval, dag.concurrency AS dag_concurrency, dag.has_task_concurrency_limits AS dag_has_task_concurrency_limits, dag.next_dagrun AS dag_next_dagrun, dag.next_dagrun_create_after AS dag_next_dagrun_create_after
FROM dag
WHERE dag.is_paused IS false AND dag.is_active IS true AND dag.next_dagrun_create_after <= now() ORDER BY dag.next_dagrun_create_after
LIMIT %s FOR UPDATE]
[parameters: (10,)]
(Background on this error at: http://sqlalche.me/e/13/e3q8)
[2020-10-26 09:04:06,512] {{process_utils.py:102}} INFO - Sending Signals.SIGTERM to GPID 7437
[2020-10-26 09:04:07,029] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7762, status='terminated', started='09:04:05') (7762) terminated with exit code None
[2020-10-26 09:04:07,122] {{process_utils.py:219}} INFO - Waiting up to 5 seconds for processes to exit...
[2020-10-26 09:04:07,126] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7774, status='terminated', started='09:04:05') (7774) terminated with exit code None
[2020-10-26 09:04:07,128] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7437, status='terminated', exitcode=0, started='09:03:54') (7437) terminated with exit code 0
[2020-10-26 09:04:07,128] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7775, status='terminated', started='09:04:05') (7775) terminated with exit code None
[2020-10-26 09:04:07,129] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7773, status='terminated', started='09:04:05') (7773) terminated with exit code None
[2020-10-26 09:04:07,129] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7782, status='terminated', started='09:04:05') (7782) terminated with exit code None

# Fixed code
def _executable_task_instances_to_queued(
    self, max_tis: int, session: Session = None
) -> List[TI]:
    """
    Finds TIs that are ready for execution with respect to pool limits,
    dag concurrency, executor state, and priority.

    :param max_tis: Maximum number of TIs to queue in this loop.
    :type max_tis: int
    :return: list[airflow.models.TaskInstance]
    """
    executable_tis: List[TI] = []

    # Get the pool settings. We get a lock on the pool rows, treating this as a "critical section"
    # Throws an exception if lock cannot be obtained, rather than blocking
    pools = models.Pool.slots_stats(lock_rows=True, session=session)

    # If the pools are full, there is no point doing anything!
    # If _somehow_ the pool is overfull, don't let the limit go negative - it breaks SQL
    pool_slots_free = max(0, sum(pool["open"] for pool in pools.values()))

    if pool_slots_free == 0:
        self.log.debug("All pools are full!")
        return executable_tis

    max_tis = min(max_tis, pool_slots_free)

    # Get all task instances associated with scheduled
    # DagRuns which are not backfilled, in the given states,
    # and the dag is not paused
    query = (
        session.query(TI)
        .outerjoin(TI.dag_run)
        .filter(or_(DR.run_id.is_(None), DR.run_type != DagRunType.BACKFILL_JOB))
        .join(TI.dag_model)
        .filter(not_(DM.is_paused))
        .filter(TI.state == State.SCHEDULED)
        .options(selectinload("dag_model"))
        .limit(max_tis)
    )

    task_instances_to_examine: List[TI] = with_row_locks(
        query,
        of=TI,
        session=session,
        **skip_locked(session=session),
    ).all()
    # TODO[HA]: This was wrong before anyway, as it only looked at a sub-set of dags, not everything.
    # Stats.gauge('scheduler.tasks.pending', len(task_instances_to_examine))

    if len(task_instances_to_examine) == 0:
        self.log.debug("No tasks to consider for execution.")
        return executable_tis

    # Put one task instance on each line
    task_instance_str = "\n\t".join([repr(x) for x in task_instances_to_examine])
    self.log.info(
        "%s tasks up for execution:\n\t%s",
        len(task_instances_to_examine),
        task_instance_str,
    )

    pool_to_task_instances: DefaultDict[str, List[models.Pool]] = defaultdict(list)
    for task_instance in task_instances_to_examine:
        pool_to_task_instances[task_instance.pool].append(task_instance)

    # dag_id to # of running tasks and (dag_id, task_id) to # of running tasks.
    dag_concurrency_map: DefaultDict[str, int]
    task_concurrency_map: DefaultDict[Tuple[str, str], int]
    dag_concurrency_map, task_concurrency_map = self.__get_concurrency_maps(
        states=list(EXECUTION_STATES), session=session
    )

    num_tasks_in_executor = 0
    # Number of tasks that cannot be scheduled because of no open slot in pool
    num_starving_tasks_total = 0

    # Go through each pool, and queue up a task for execution if there are
    # any open slots in the pool.
    # pylint: disable=too-many-nested-blocks
    for pool, task_instances in pool_to_task_instances.items():
        pool_name = pool
        if pool not in pools:
            self.log.warning(
                "Tasks using non-existent pool '%s' will not be scheduled", pool
            )
            continue

        open_slots = pools[pool]["open"]

        num_ready = len(task_instances)
        self.log.info(
            "Figuring out tasks to run in Pool(name=%s) with %s open slots "
            "and %s task instances ready to be queued",
            pool,
            open_slots,
            num_ready,
        )

        priority_sorted_task_instances = sorted(
            task_instances, key=lambda ti: (-ti.priority_weight, ti.execution_date)
        )

        num_starving_tasks = 0
        for current_index, task_instance in enumerate(priority_sorted_task_instances):
            if open_slots <= 0:
                self.log.info(
                    "Not scheduling since there are %s open slots in pool %s",
                    open_slots,
                    pool,
                )
                # Can't schedule any more since there are no more open slots.
                num_unhandled = len(priority_sorted_task_instances) - current_index
                num_starving_tasks += num_unhandled
                num_starving_tasks_total += num_unhandled
                break

            # Check to make sure that the task concurrency of the DAG hasn't been
            # reached.
            dag_id = task_instance.dag_id

            current_dag_concurrency = dag_concurrency_map[dag_id]
            dag_concurrency_limit = task_instance.dag_model.concurrency
            self.log.info(
                "DAG %s has %s/%s running and queued tasks",
                dag_id,
                current_dag_concurrency,
                dag_concurrency_limit,
            )
            if current_dag_concurrency >= dag_concurrency_limit:
                self.log.info(
                    "Not executing %s since the number of tasks running or queued "
                    "from DAG %s is >= to the DAG's task concurrency limit of %s",
                    task_instance,
                    dag_id,
                    dag_concurrency_limit,
                )
                continue

            task_concurrency_limit: Optional[int] = None
            if task_instance.dag_model.has_task_concurrency_limits:
                # Many dags don't have a task_concurrency, so where we can avoid loading the full
                # serialized DAG the better.
                serialized_dag = self.dagbag.get_dag(dag_id, session=session)
                if serialized_dag.has_task(task_instance.task_id):
                    task_concurrency_limit = serialized_dag.get_task(
                        task_instance.task_id
                    ).task_concurrency

                if task_concurrency_limit is not None:
                    current_task_concurrency = task_concurrency_map[
                        (task_instance.dag_id, task_instance.task_id)
                    ]

                    if current_task_concurrency >= task_concurrency_limit:
                        self.log.info(
                            "Not executing %s since the task concurrency for"
                            " this task has been reached.",
                            task_instance,
                        )
                        continue

            if task_instance.pool_slots > open_slots:
                self.log.info(
                    "Not executing %s since it requires %s slots "
                    "but there are %s open slots in the pool %s.",
                    task_instance,
                    task_instance.pool_slots,
                    open_slots,
                    pool,
                )
                num_starving_tasks += 1
                num_starving_tasks_total += 1
                # Though we can execute tasks with lower priority if there's enough room
                continue

            executable_tis.append(task_instance)
            open_slots -= task_instance.pool_slots
            dag_concurrency_map[dag_id] += 1
            task_concurrency_map[(task_instance.dag_id, task_instance.task_id)] += 1

        Stats.gauge(f"pool.starving_tasks.{pool_name}", num_starving_tasks)

    Stats.gauge("scheduler.tasks.starving", num_starving_tasks_total)
    Stats.gauge("scheduler.tasks.running", num_tasks_in_executor)
    Stats.gauge("scheduler.tasks.executable", len(executable_tis))

    task_instance_str = "\n\t".join([repr(x) for x in executable_tis])
    self.log.info(
        "Setting the following tasks to queued state:\n\t%s", task_instance_str
    )

    # set TIs to queued state
    filter_for_tis = TI.filter_for_tis(executable_tis)
    session.query(TI).filter(filter_for_tis).update(
        # TODO[ha]: should we use func.now()? How does that work with DB timezone on mysql when it's not
        # UTC?
        {
            TI.state: State.QUEUED,
            TI.queued_dttm: timezone.utcnow(),
            TI.queued_by_job_id: self.id,
        },
        synchronize_session=False,
    )

    for ti in executable_tis:
        make_transient(ti)
    return executable_tis