# Buggy code
def _do_scheduling(self, session) -> int:
    """
    This function is where the main scheduling decisions take places. It:

    - Creates any necessary DAG runs by examining the next_dagrun_create_after column of DagModel

      Since creating Dag Runs is a relatively time consuming process, we select only 10 dags by default
      (configurable via ``scheduler.max_dagruns_to_create_per_loop`` setting) - putting this higher will
      mean one scheduler could spend a chunk of time creating dag runs, and not ever get around to
      scheduling tasks.

    - Finds the "next n oldest" running DAG Runs to examine for scheduling (n=20 by default, configurable
      via ``scheduler.max_dagruns_per_loop_to_schedule`` config setting) and tries to progress state (TIs
      to SCHEDULED, or DagRuns to SUCCESS/FAILURE etc)

      By "next oldest", we mean hasn't been examined/scheduled in the most time.

      The reason we don't select all dagruns at once because the rows are selected with row locks, meaning
      that only one scheduler can "process them", even it it is waiting behind other dags. Increasing this
      limit will allow more throughput for smaller DAGs but will likely slow down throughput for larger
      (>500 tasks.) DAGs

    - Then, via a Critical Section (locking the rows of the Pool model) we queue tasks, and then send them
      to the executor.

      See docs of _critical_section_execute_task_instances for more.

    :return: Number of TIs enqueued in this iteration
    :rtype: int
    """
    # Put a check in place to make sure we don't commit unexpectedly
    with prohibit_commit(session) as guard:
        if settings.USE_JOB_SCHEDULE:
            query = DagModel.dags_needing_dagruns(session)
            self._create_dag_runs(query.all(), session)

            # commit the session - Release the write lock on DagModel table.
            guard.commit()
            # END: create dagruns

        dag_runs = DagRun.next_dagruns_to_examine(session)

        # Bulk fetch the currently active dag runs for the dags we are
        # examining, rather than making one query per DagRun

        # TODO: This query is probably horribly inefficient (though there is an
        # index on (dag_id,state)). It is to deal with the case when a user
        # clears more than max_active_runs older tasks -- we don't want the
        # scheduler to suddenly go and start running tasks from all of the
        # runs. (AIRFLOW-137/GH #1442)
        #
        # The longer term fix would be to have `clear` do this, and put DagRuns
        # in to the queued state, then take DRs out of queued before creating
        # any new ones

        # Build up a set of execution_dates that are "active" for a given
        # dag_id -- only tasks from those runs will be scheduled.
        active_runs_by_dag_id = defaultdict(set)

        query = (
            session.query(
                TI.dag_id,
                TI.execution_date,
            )
            .filter(
                TI.dag_id.in_(list({dag_run.dag_id for dag_run in dag_runs})),
                TI.state.notin_(list(State.finished) + [State.REMOVED]),
            )
            .group_by(TI.dag_id, TI.execution_date)
        )

        for dag_id, execution_date in query:
            active_runs_by_dag_id[dag_id].add(execution_date)

        for dag_run in dag_runs:
            # Use try_except to not stop the Scheduler when a Serialized DAG is not found
            # This takes care of Dynamic DAGs especially
            # SerializedDagNotFound should not happen here in the same loop because the DagRun would
            # not be created in self._create_dag_runs if Serialized DAG does not exist
            # But this would take care of the scenario when the Scheduler is restarted after DagRun is
            # created and the DAG is deleted / renamed
            try:
                self._schedule_dag_run(
                    dag_run, active_runs_by_dag_id.get(dag_run.dag_id, set()), session
                )
            except SerializedDagNotFound:
                self.log.exception(
                    "DAG '%s' not found in serialized_dag table", dag_run.dag_id
                )
                continue

        guard.commit()

        # Without this, the session has an invalid view of the DB
        session.expunge_all()
        # END: schedule TIs

        try:
            if self.executor.slots_available <= 0:
                # We know we can't do anything here, so don't even try!
                self.log.debug("Executor full, skipping critical section")
                return 0

            timer = Stats.timer("scheduler.critical_section_duration")
            timer.start()

            # Find anything TIs in state SCHEDULED, try to QUEUE it (send it to the executor)
            num_queued_tis = self._critical_section_execute_task_instances(
                session=session
            )

            # Make sure we only sent this metric if we obtained the lock, otherwise we'll skew the
            # metric, way down
            timer.stop(send=True)
        except OperationalError as e:
            timer.stop(send=False)

            if is_lock_not_available_error(error=e):
                self.log.debug("Critical section lock held by another Scheduler")
                Stats.incr("scheduler.critical_section_busy")
                session.rollback()
                return 0
            raise

        guard.commit()
        return num_queued_tis


# Traceback (_mysql_exceptions.OperationalError)
[2020-10-26 09:03:54,608] {{settings.py:49}} INFO - Configured default timezone Timezone('UTC')
[2020-10-26 09:04:05,467] {{scheduler_job.py:1327}} ERROR - Exception when executing SchedulerJob._run_scheduler_loop
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context
self.dialect.do_execute(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 255, in execute
self.errorhandler(self, exc, value)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/connections.py", line 50, in defaulterrorhandler
raise errorvalue
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 252, in execute
res = self._query(query)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 379, in _query
self._do_get_result(db)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 182, in _do_get_result
self._result = result = self._get_result()
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 411, in _get_result
return self._get_db().store_result()
_mysql_exceptions.OperationalError: (1213, 'Deadlock found when trying to get lock; try restarting transaction')

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py", line 1308, in _execute
self._run_scheduler_loop()
File "/home/airflow/.local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py", line 1379, in _run_scheduler_loop
num_queued_tis = self._do_scheduling(session)
File "/home/airflow/.local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py", line 1451, in _do_scheduling
self._create_dag_runs(query.all(), session)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3341, in all
return list(self)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3503, in __iter__
return self._execute_and_instances(context)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3528, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1014, in execute
return meth(self, multiparams, params)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1127, in _execute_clauseelement
ret = self._execute_context(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1317, in _execute_context
self._handle_dbapi_exception(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1511, in _handle_dbapi_exception
util.raise_(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
raise exception
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context
self.dialect.do_execute(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 255, in execute
self.errorhandler(self, exc, value)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/connections.py", line 50, in defaulterrorhandler
raise errorvalue
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 252, in execute
res = self._query(query)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 379, in _query
self._do_get_result(db)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 182, in _do_get_result
self._result = result = self._get_result()
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 411, in _get_result
return self._get_db().store_result()
sqlalchemy.exc.OperationalError: (_mysql_exceptions.OperationalError) (1213, 'Deadlock found when trying to get lock; try restarting transaction')
[SQL: SELECT dag.dag_id AS dag_dag_id, dag.root_dag_id AS dag_root_dag_id, dag.is_paused AS dag_is_paused, dag.is_subdag AS dag_is_subdag, dag.is_active AS dag_is_active, dag.last_scheduler_run AS dag_last_scheduler_run, dag.last_pickled AS dag_last_pickled, dag.last_expired AS dag_last_expired, dag.scheduler_lock AS dag_scheduler_lock, dag.pickle_id AS dag_pickle_id, dag.fileloc AS dag_fileloc, dag.owners AS dag_owners, dag.description AS dag_description, dag.default_view AS dag_default_view, dag.schedule_interval AS dag_schedule_interval, dag.concurrency AS dag_concurrency, dag.has_task_concurrency_limits AS dag_has_task_concurrency_limits, dag.next_dagrun AS dag_next_dagrun, dag.next_dagrun_create_after AS dag_next_dagrun_create_after
FROM dag
WHERE dag.is_paused IS false AND dag.is_active IS true AND dag.next_dagrun_create_after <= now() ORDER BY dag.next_dagrun_create_after
LIMIT %s FOR UPDATE]
[parameters: (10,)]
(Background on this error at: http://sqlalche.me/e/13/e3q8)
[2020-10-26 09:04:06,512] {{process_utils.py:102}} INFO - Sending Signals.SIGTERM to GPID 7437
[2020-10-26 09:04:07,029] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7762, status='terminated', started='09:04:05') (7762) terminated with exit code None
[2020-10-26 09:04:07,122] {{process_utils.py:219}} INFO - Waiting up to 5 seconds for processes to exit...
[2020-10-26 09:04:07,126] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7774, status='terminated', started='09:04:05') (7774) terminated with exit code None
[2020-10-26 09:04:07,128] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7437, status='terminated', exitcode=0, started='09:03:54') (7437) terminated with exit code 0
[2020-10-26 09:04:07,128] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7775, status='terminated', started='09:04:05') (7775) terminated with exit code None
[2020-10-26 09:04:07,129] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7773, status='terminated', started='09:04:05') (7773) terminated with exit code None
[2020-10-26 09:04:07,129] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7782, status='terminated', started='09:04:05') (7782) terminated with exit code None

# Fixed code
def _do_scheduling(self, session) -> int:
    """
    This function is where the main scheduling decisions take places. It:

    - Creates any necessary DAG runs by examining the next_dagrun_create_after column of DagModel

      Since creating Dag Runs is a relatively time consuming process, we select only 10 dags by default
      (configurable via ``scheduler.max_dagruns_to_create_per_loop`` setting) - putting this higher will
      mean one scheduler could spend a chunk of time creating dag runs, and not ever get around to
      scheduling tasks.

    - Finds the "next n oldest" running DAG Runs to examine for scheduling (n=20 by default, configurable
      via ``scheduler.max_dagruns_per_loop_to_schedule`` config setting) and tries to progress state (TIs
      to SCHEDULED, or DagRuns to SUCCESS/FAILURE etc)

      By "next oldest", we mean hasn't been examined/scheduled in the most time.

      The reason we don't select all dagruns at once because the rows are selected with row locks, meaning
      that only one scheduler can "process them", even it it is waiting behind other dags. Increasing this
      limit will allow more throughput for smaller DAGs but will likely slow down throughput for larger
      (>500 tasks.) DAGs

    - Then, via a Critical Section (locking the rows of the Pool model) we queue tasks, and then send them
      to the executor.

      See docs of _critical_section_execute_task_instances for more.

    :return: Number of TIs enqueued in this iteration
    :rtype: int
    """
    # Put a check in place to make sure we don't commit unexpectedly
    with prohibit_commit(session) as guard:
        if settings.USE_JOB_SCHEDULE:
            self._create_dagruns_for_dags(guard, session)

        dag_runs = self._get_next_dagruns_to_examine(session)
        # Bulk fetch the currently active dag runs for the dags we are
        # examining, rather than making one query per DagRun

        # TODO: This query is probably horribly inefficient (though there is an
        # index on (dag_id,state)). It is to deal with the case when a user
        # clears more than max_active_runs older tasks -- we don't want the
        # scheduler to suddenly go and start running tasks from all of the
        # runs. (AIRFLOW-137/GH #1442)
        #
        # The longer term fix would be to have `clear` do this, and put DagRuns
        # in to the queued state, then take DRs out of queued before creating
        # any new ones

        # Build up a set of execution_dates that are "active" for a given
        # dag_id -- only tasks from those runs will be scheduled.
        active_runs_by_dag_id = defaultdict(set)

        query = (
            session.query(
                TI.dag_id,
                TI.execution_date,
            )
            .filter(
                TI.dag_id.in_(list({dag_run.dag_id for dag_run in dag_runs})),
                TI.state.notin_(list(State.finished) + [State.REMOVED]),
            )
            .group_by(TI.dag_id, TI.execution_date)
        )

        for dag_id, execution_date in query:
            active_runs_by_dag_id[dag_id].add(execution_date)

        for dag_run in dag_runs:
            # Use try_except to not stop the Scheduler when a Serialized DAG is not found
            # This takes care of Dynamic DAGs especially
            # SerializedDagNotFound should not happen here in the same loop because the DagRun would
            # not be created in self._create_dag_runs if Serialized DAG does not exist
            # But this would take care of the scenario when the Scheduler is restarted after DagRun is
            # created and the DAG is deleted / renamed
            try:
                self._schedule_dag_run(
                    dag_run, active_runs_by_dag_id.get(dag_run.dag_id, set()), session
                )
            except SerializedDagNotFound:
                self.log.exception(
                    "DAG '%s' not found in serialized_dag table", dag_run.dag_id
                )
                continue

        guard.commit()

        # Without this, the session has an invalid view of the DB
        session.expunge_all()
        # END: schedule TIs

        try:
            if self.executor.slots_available <= 0:
                # We know we can't do anything here, so don't even try!
                self.log.debug("Executor full, skipping critical section")
                return 0

            timer = Stats.timer("scheduler.critical_section_duration")
            timer.start()

            # Find anything TIs in state SCHEDULED, try to QUEUE it (send it to the executor)
            num_queued_tis = self._critical_section_execute_task_instances(
                session=session
            )

            # Make sure we only sent this metric if we obtained the lock, otherwise we'll skew the
            # metric, way down
            timer.stop(send=True)
        except OperationalError as e:
            timer.stop(send=False)

            if is_lock_not_available_error(error=e):
                self.log.debug("Critical section lock held by another Scheduler")
                Stats.incr("scheduler.critical_section_busy")
                session.rollback()
                return 0
            raise

        guard.commit()
        return num_queued_tis