# Buggy code
def adopt_or_reset_orphaned_tasks(self, session: Session = None):
    """
    Reset any TaskInstance still in QUEUED or SCHEDULED states that were
    enqueued by a SchedulerJob that is no longer running.

    :return: the number of TIs reset
    :rtype: int
    """
    self.log.info("Resetting orphaned tasks for active dag runs")
    timeout = conf.getint("scheduler", "scheduler_health_check_threshold")

    num_failed = (
        session.query(SchedulerJob)
        .filter(
            SchedulerJob.state == State.RUNNING,
            SchedulerJob.latest_heartbeat
            < (timezone.utcnow() - timedelta(seconds=timeout)),
        )
        .update({"state": State.FAILED})
    )

    if num_failed:
        self.log.info("Marked %d SchedulerJob instances as failed", num_failed)
        Stats.incr(self.__class__.__name__.lower() + "_end", num_failed)

    resettable_states = [State.SCHEDULED, State.QUEUED, State.RUNNING]
    query = (
        session.query(TI)
        .filter(TI.state.in_(resettable_states))
        # outerjoin is because we didn't use to have queued_by_job
        # set, so we need to pick up anything pre upgrade. This (and the
        # "or queued_by_job_id IS NONE") can go as soon as scheduler HA is
        # released.
        .outerjoin(TI.queued_by_job)
        .filter(or_(TI.queued_by_job_id.is_(None), SchedulerJob.state != State.RUNNING))
        .join(TI.dag_run)
        .filter(
            DagRun.run_type != DagRunType.BACKFILL_JOB,
            # pylint: disable=comparison-with-callable
            DagRun.state == State.RUNNING,
        )
        .options(load_only(TI.dag_id, TI.task_id, TI.execution_date))
    )

    # Lock these rows, so that another scheduler can't try and adopt these too
    tis_to_reset_or_adopt = with_row_locks(
        query, of=TI, session=session, **skip_locked(session=session)
    ).all()
    to_reset = self.executor.try_adopt_task_instances(tis_to_reset_or_adopt)

    reset_tis_message = []
    for ti in to_reset:
        reset_tis_message.append(repr(ti))
        ti.state = State.NONE
        ti.queued_by_job_id = None

    for ti in set(tis_to_reset_or_adopt) - set(to_reset):
        ti.queued_by_job_id = self.id

    Stats.incr("scheduler.orphaned_tasks.cleared", len(to_reset))
    Stats.incr(
        "scheduler.orphaned_tasks.adopted", len(tis_to_reset_or_adopt) - len(to_reset)
    )

    if to_reset:
        task_instance_str = "\n\t".join(reset_tis_message)
        self.log.info(
            "Reset the following %s orphaned TaskInstances:\n\t%s",
            len(to_reset),
            task_instance_str,
        )

    # Issue SQL/finish "Unit of Work", but let @provide_session commit (or if passed a session, let caller
    # decide when to commit
    session.flush()
    return len(to_reset)


# Traceback (_mysql_exceptions.OperationalError)
[2020-10-26 09:03:54,608] {{settings.py:49}} INFO - Configured default timezone Timezone('UTC')
[2020-10-26 09:04:05,467] {{scheduler_job.py:1327}} ERROR - Exception when executing SchedulerJob._run_scheduler_loop
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context
self.dialect.do_execute(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 255, in execute
self.errorhandler(self, exc, value)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/connections.py", line 50, in defaulterrorhandler
raise errorvalue
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 252, in execute
res = self._query(query)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 379, in _query
self._do_get_result(db)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 182, in _do_get_result
self._result = result = self._get_result()
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 411, in _get_result
return self._get_db().store_result()
_mysql_exceptions.OperationalError: (1213, 'Deadlock found when trying to get lock; try restarting transaction')

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py", line 1308, in _execute
self._run_scheduler_loop()
File "/home/airflow/.local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py", line 1379, in _run_scheduler_loop
num_queued_tis = self._do_scheduling(session)
File "/home/airflow/.local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py", line 1451, in _do_scheduling
self._create_dag_runs(query.all(), session)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3341, in all
return list(self)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3503, in __iter__
return self._execute_and_instances(context)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3528, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1014, in execute
return meth(self, multiparams, params)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1127, in _execute_clauseelement
ret = self._execute_context(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1317, in _execute_context
self._handle_dbapi_exception(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1511, in _handle_dbapi_exception
util.raise_(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
raise exception
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context
self.dialect.do_execute(
File "/home/airflow/.local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 255, in execute
self.errorhandler(self, exc, value)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/connections.py", line 50, in defaulterrorhandler
raise errorvalue
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 252, in execute
res = self._query(query)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 379, in _query
self._do_get_result(db)
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 182, in _do_get_result
self._result = result = self._get_result()
File "/home/airflow/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 411, in _get_result
return self._get_db().store_result()
sqlalchemy.exc.OperationalError: (_mysql_exceptions.OperationalError) (1213, 'Deadlock found when trying to get lock; try restarting transaction')
[SQL: SELECT dag.dag_id AS dag_dag_id, dag.root_dag_id AS dag_root_dag_id, dag.is_paused AS dag_is_paused, dag.is_subdag AS dag_is_subdag, dag.is_active AS dag_is_active, dag.last_scheduler_run AS dag_last_scheduler_run, dag.last_pickled AS dag_last_pickled, dag.last_expired AS dag_last_expired, dag.scheduler_lock AS dag_scheduler_lock, dag.pickle_id AS dag_pickle_id, dag.fileloc AS dag_fileloc, dag.owners AS dag_owners, dag.description AS dag_description, dag.default_view AS dag_default_view, dag.schedule_interval AS dag_schedule_interval, dag.concurrency AS dag_concurrency, dag.has_task_concurrency_limits AS dag_has_task_concurrency_limits, dag.next_dagrun AS dag_next_dagrun, dag.next_dagrun_create_after AS dag_next_dagrun_create_after
FROM dag
WHERE dag.is_paused IS false AND dag.is_active IS true AND dag.next_dagrun_create_after <= now() ORDER BY dag.next_dagrun_create_after
LIMIT %s FOR UPDATE]
[parameters: (10,)]
(Background on this error at: http://sqlalche.me/e/13/e3q8)
[2020-10-26 09:04:06,512] {{process_utils.py:102}} INFO - Sending Signals.SIGTERM to GPID 7437
[2020-10-26 09:04:07,029] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7762, status='terminated', started='09:04:05') (7762) terminated with exit code None
[2020-10-26 09:04:07,122] {{process_utils.py:219}} INFO - Waiting up to 5 seconds for processes to exit...
[2020-10-26 09:04:07,126] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7774, status='terminated', started='09:04:05') (7774) terminated with exit code None
[2020-10-26 09:04:07,128] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7437, status='terminated', exitcode=0, started='09:03:54') (7437) terminated with exit code 0
[2020-10-26 09:04:07,128] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7775, status='terminated', started='09:04:05') (7775) terminated with exit code None
[2020-10-26 09:04:07,129] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7773, status='terminated', started='09:04:05') (7773) terminated with exit code None
[2020-10-26 09:04:07,129] {{process_utils.py:68}} INFO - Process psutil.Process(pid=7782, status='terminated', started='09:04:05') (7782) terminated with exit code None

# Fixed code
def adopt_or_reset_orphaned_tasks(self, session: Session = None):
    """
    Reset any TaskInstance still in QUEUED or SCHEDULED states that were
    enqueued by a SchedulerJob that is no longer running.

    :return: the number of TIs reset
    :rtype: int
    """
    self.log.info("Resetting orphaned tasks for active dag runs")
    timeout = conf.getint("scheduler", "scheduler_health_check_threshold")

    for attempt in run_with_db_retries(logger=self.log):
        with attempt:
            self.log.debug(
                "Running SchedulerJob.adopt_or_reset_orphaned_tasks with retries. Try %d of %d",
                attempt.retry_state.attempt_number,
                settings.MAX_DB_RETRIES,
            )
            self.log.debug("Calling SchedulerJob.adopt_or_reset_orphaned_tasks method")
            try:
                num_failed = (
                    session.query(SchedulerJob)
                    .filter(
                        SchedulerJob.state == State.RUNNING,
                        SchedulerJob.latest_heartbeat
                        < (timezone.utcnow() - timedelta(seconds=timeout)),
                    )
                    .update({"state": State.FAILED})
                )

                if num_failed:
                    self.log.info(
                        "Marked %d SchedulerJob instances as failed", num_failed
                    )
                    Stats.incr(self.__class__.__name__.lower() + "_end", num_failed)

                resettable_states = [State.SCHEDULED, State.QUEUED, State.RUNNING]
                query = (
                    session.query(TI)
                    .filter(TI.state.in_(resettable_states))
                    # outerjoin is because we didn't use to have queued_by_job
                    # set, so we need to pick up anything pre upgrade. This (and the
                    # "or queued_by_job_id IS NONE") can go as soon as scheduler HA is
                    # released.
                    .outerjoin(TI.queued_by_job)
                    .filter(
                        or_(
                            TI.queued_by_job_id.is_(None),
                            SchedulerJob.state != State.RUNNING,
                        )
                    )
                    .join(TI.dag_run)
                    .filter(
                        DagRun.run_type != DagRunType.BACKFILL_JOB,
                        # pylint: disable=comparison-with-callable
                        DagRun.state == State.RUNNING,
                    )
                    .options(load_only(TI.dag_id, TI.task_id, TI.execution_date))
                )

                # Lock these rows, so that another scheduler can't try and adopt these too
                tis_to_reset_or_adopt = with_row_locks(
                    query, of=TI, session=session, **skip_locked(session=session)
                ).all()
                to_reset = self.executor.try_adopt_task_instances(tis_to_reset_or_adopt)

                reset_tis_message = []
                for ti in to_reset:
                    reset_tis_message.append(repr(ti))
                    ti.state = State.NONE
                    ti.queued_by_job_id = None

                for ti in set(tis_to_reset_or_adopt) - set(to_reset):
                    ti.queued_by_job_id = self.id

                Stats.incr("scheduler.orphaned_tasks.cleared", len(to_reset))
                Stats.incr(
                    "scheduler.orphaned_tasks.adopted",
                    len(tis_to_reset_or_adopt) - len(to_reset),
                )

                if to_reset:
                    task_instance_str = "\n\t".join(reset_tis_message)
                    self.log.info(
                        "Reset the following %s orphaned TaskInstances:\n\t%s",
                        len(to_reset),
                        task_instance_str,
                    )

                # Issue SQL/finish "Unit of Work", but let @provide_session
                # commit (or if passed a session, let caller decide when to commit
                session.flush()
            except OperationalError:
                session.rollback()
                raise

    return len(to_reset)