if execution_date and execution_date < self.execution_date:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'.format(<EOL>self.execution_date, execution_date))<EOL><DEDENT>XCom.set(<EOL>key=key,<EOL>value=value,<EOL>task_id=self.task_id,<EOL>dag_id=self.dag_id,<EOL>execution_date=execution_date or self.execution_date)<EOL>

Make an XCom available for tasks to pull.

:param key: A key for the XCom
:type key: str
:param value: A value for the XCom. The value is pickled and stored
    in the database.
:type value: any pickleable object
:param execution_date: if provided, the XCom will not be visible until
    this date. This can be used, for example, to send a message to a
    task on a future date without it being immediately visible.
:type execution_date: datetime


if dag_id is None:<EOL><INDENT>dag_id = self.dag_id<EOL><DEDENT>pull_fn = functools.partial(<EOL>XCom.get_one,<EOL>execution_date=self.execution_date,<EOL>key=key,<EOL>dag_id=dag_id,<EOL>include_prior_dates=include_prior_dates)<EOL>if is_container(task_ids):<EOL><INDENT>return tuple(pull_fn(task_id=t) for t in task_ids)<EOL><DEDENT>else:<EOL><INDENT>return pull_fn(task_id=task_ids)<EOL><DEDENT>

Pull XComs that optionally meet certain criteria.

The default value for `key` limits the search to XComs
that were returned by other tasks (as opposed to those that were pushed
manually). To remove this filter, pass key=None (or any desired value).

If a single task_id string is provided, the result is the value of the
most recent matching XCom from that task_id. If multiple task_ids are
provided, a tuple of matching values is returned. None is returned
whenever no matches are found.

:param key: A key for the XCom. If provided, only XComs with matching
    keys will be returned. The default key is 'return_value', also
    available as a constant XCOM_RETURN_KEY. This key is automatically
    given to XComs returned by tasks (as opposed to being pushed
    manually). To remove the filter, pass key=None.
:type key: str
:param task_ids: Only XComs from tasks with matching ids will be
    pulled. Can pass None to remove the filter.
:type task_ids: str or iterable of strings (representing task_ids)
:param dag_id: If provided, only pulls XComs from this DAG.
    If None (default), the DAG of the calling task is used.
:type dag_id: str
:param include_prior_dates: If False, only XComs from the current
    execution_date are returned. If True, XComs from previous dates
    are returned as well.
:type include_prior_dates: bool


self.raw = raw<EOL>self._set_context(self)<EOL>


from airflow.models.taskinstance importTaskInstance as TI  <EOL>used_slots = session.query(func.count()).filter(TI.pool == self.pool).filter(<EOL>TI.state.in_([State.RUNNING, State.QUEUED])).scalar()<EOL>return self.slots - used_slots<EOL>

Returns the number of slots open at the moment


DR = DagRun<EOL>exec_date = func.cast(self.execution_date, DateTime)<EOL>dr = session.query(DR).filter(<EOL>DR.dag_id == self.dag_id,<EOL>func.cast(DR.execution_date, DateTime) == exec_date,<EOL>DR.run_id == self.run_id<EOL>).one()<EOL>self.id = dr.id<EOL>self.state = dr.state<EOL>

Reloads the current dagrun from the database
:param session: database session


@staticmethod<EOL><INDENT>@provide_session<EOL>def find(dag_id=None, run_id=None, execution_date=None,<EOL>state=None, external_trigger=None, no_backfills=False,<EOL>session=None):<DEDENT>

DR = DagRun<EOL>qry = session.query(DR)<EOL>if dag_id:<EOL><INDENT>qry = qry.filter(DR.dag_id == dag_id)<EOL><DEDENT>if run_id:<EOL><INDENT>qry = qry.filter(DR.run_id == run_id)<EOL><DEDENT>if execution_date:<EOL><INDENT>if isinstance(execution_date, list):<EOL><INDENT>qry = qry.filter(DR.execution_date.in_(execution_date))<EOL><DEDENT>else:<EOL><INDENT>qry = qry.filter(DR.execution_date == execution_date)<EOL><DEDENT><DEDENT>if state:<EOL><INDENT>qry = qry.filter(DR.state == state)<EOL><DEDENT>if external_trigger is not None:<EOL><INDENT>qry = qry.filter(DR.external_trigger == external_trigger)<EOL><DEDENT>if no_backfills:<EOL><INDENT>from airflow.jobs import BackfillJob<EOL>qry = qry.filter(DR.run_id.notlike(BackfillJob.ID_PREFIX + '<STR_LIT:%>'))<EOL><DEDENT>dr = qry.order_by(DR.execution_date).all()<EOL>return dr<EOL>

Returns a set of dag runs for the given search criteria.

:param dag_id: the dag_id to find dag runs for
:type dag_id: int, list
:param run_id: defines the the run id for this dag run
:type run_id: str
:param execution_date: the execution date
:type execution_date: datetime.datetime
:param state: the state of the dag run
:type state: airflow.utils.state.State
:param external_trigger: whether this dag run is externally triggered
:type external_trigger: bool
:param no_backfills: return no backfills (True), return all (False).
    Defaults to False
:type no_backfills: bool
:param session: database session
:type session: sqlalchemy.orm.session.Session


@provide_session<EOL><INDENT>def get_task_instances(self, state=None, session=None):<DEDENT>

from airflow.models.taskinstance import TaskInstance  <EOL>tis = session.query(TaskInstance).filter(<EOL>TaskInstance.dag_id == self.dag_id,<EOL>TaskInstance.execution_date == self.execution_date,<EOL>)<EOL>if state:<EOL><INDENT>if isinstance(state, six.string_types):<EOL><INDENT>tis = tis.filter(TaskInstance.state == state)<EOL><DEDENT>else:<EOL><INDENT>if None in state:<EOL><INDENT>tis = tis.filter(<EOL>or_(TaskInstance.state.in_(state),<EOL>TaskInstance.state.is_(None))<EOL>)<EOL><DEDENT>else:<EOL><INDENT>tis = tis.filter(TaskInstance.state.in_(state))<EOL><DEDENT><DEDENT><DEDENT>if self.dag and self.dag.partial:<EOL><INDENT>tis = tis.filter(TaskInstance.task_id.in_(self.dag.task_ids))<EOL><DEDENT>return tis.all()<EOL>

Returns the task instances for this dag run


@provide_session<EOL><INDENT>def get_task_instance(self, task_id, session=None):<DEDENT>

from airflow.models.taskinstance import TaskInstance  <EOL>TI = TaskInstance<EOL>ti = session.query(TI).filter(<EOL>TI.dag_id == self.dag_id,<EOL>TI.execution_date == self.execution_date,<EOL>TI.task_id == task_id<EOL>).first()<EOL>return ti<EOL>

Returns the task instance specified by task_id for this dag run

:param task_id: the task id


if not self.dag:<EOL><INDENT>raise AirflowException("<STR_LIT>"<EOL>.format(self))<EOL><DEDENT>return self.dag<EOL>

Returns the Dag associated with this DagRun.

:return: DAG


return session.query(DagRun).filter(<EOL>DagRun.dag_id == self.dag_id,<EOL>DagRun.execution_date < self.execution_date<EOL>).order_by(<EOL>DagRun.execution_date.desc()<EOL>).first()<EOL>


dag = self.get_dag()<EOL>return session.query(DagRun).filter(<EOL>DagRun.dag_id == self.dag_id,<EOL>DagRun.execution_date == dag.previous_schedule(self.execution_date)<EOL>).first()<EOL>

The previous, SCHEDULED DagRun, if there is one


dag = self.get_dag()<EOL>tis = self.get_task_instances(session=session)<EOL>self.log.debug("<STR_LIT>", self, len(tis))<EOL>for ti in list(tis):<EOL><INDENT>if ti.state == State.REMOVED:<EOL><INDENT>tis.remove(ti)<EOL><DEDENT>else:<EOL><INDENT>ti.task = dag.get_task(ti.task_id)<EOL><DEDENT><DEDENT>start_dttm = timezone.utcnow()<EOL>unfinished_tasks = self.get_task_instances(<EOL>state=State.unfinished(),<EOL>session=session<EOL>)<EOL>none_depends_on_past = all(not t.task.depends_on_past for t in unfinished_tasks)<EOL>none_task_concurrency = all(t.task.task_concurrency is None<EOL>for t in unfinished_tasks)<EOL>if unfinished_tasks and none_depends_on_past and none_task_concurrency:<EOL><INDENT>no_dependencies_met = True<EOL>for ut in unfinished_tasks:<EOL><INDENT>old_state = ut.state<EOL>deps_met = ut.are_dependencies_met(<EOL>dep_context=DepContext(<EOL>flag_upstream_failed=True,<EOL>ignore_in_retry_period=True,<EOL>ignore_in_reschedule_period=True),<EOL>session=session)<EOL>if deps_met or old_state != ut.current_state(session=session):<EOL><INDENT>no_dependencies_met = False<EOL>break<EOL><DEDENT><DEDENT><DEDENT>duration = (timezone.utcnow() - start_dttm).total_seconds() * <NUM_LIT:1000><EOL>Stats.timing("<STR_LIT>".format(self.dag_id), duration)<EOL>root_ids = [t.task_id for t in dag.roots]<EOL>roots = [t for t in tis if t.task_id in root_ids]<EOL>if (not unfinished_tasks and<EOL>any(r.state in (State.FAILED, State.UPSTREAM_FAILED) for r in roots)):<EOL><INDENT>self.log.info('<STR_LIT>', self)<EOL>self.set_state(State.FAILED)<EOL>dag.handle_callback(self, success=False, reason='<STR_LIT>',<EOL>session=session)<EOL><DEDENT>elif not unfinished_tasks and all(r.state in (State.SUCCESS, State.SKIPPED)<EOL>for r in roots):<EOL><INDENT>self.log.info('<STR_LIT>', self)<EOL>self.set_state(State.SUCCESS)<EOL>dag.handle_callback(self, success=True, reason='<STR_LIT:success>', session=session)<EOL><DEDENT>elif (unfinished_tasks and none_depends_on_past and<EOL>none_task_concurrency and no_dependencies_met):<EOL><INDENT>self.log.info('<STR_LIT>', self)<EOL>self.set_state(State.FAILED)<EOL>dag.handle_callback(self, success=False, reason='<STR_LIT>',<EOL>session=session)<EOL><DEDENT>else:<EOL><INDENT>self.set_state(State.RUNNING)<EOL><DEDENT>self._emit_duration_stats_for_finished_state()<EOL>session.merge(self)<EOL>session.commit()<EOL>return self.state<EOL>

Determines the overall state of the DagRun based on the state
of its TaskInstances.

:return: State


from airflow.models.taskinstance import TaskInstance  <EOL>dag = self.get_dag()<EOL>tis = self.get_task_instances(session=session)<EOL>task_ids = []<EOL>for ti in tis:<EOL><INDENT>task_ids.append(ti.task_id)<EOL>task = None<EOL>try:<EOL><INDENT>task = dag.get_task(ti.task_id)<EOL><DEDENT>except AirflowException:<EOL><INDENT>if ti.state == State.REMOVED:<EOL><INDENT>pass  <EOL><DEDENT>elif self.state is not State.RUNNING and not dag.partial:<EOL><INDENT>self.log.warning("<STR_LIT>"<EOL>"<STR_LIT>".format(ti, dag))<EOL>Stats.incr(<EOL>"<STR_LIT>".format(dag.dag_id), <NUM_LIT:1>, <NUM_LIT:1>)<EOL>ti.state = State.REMOVED<EOL><DEDENT><DEDENT>is_task_in_dag = task is not None<EOL>should_restore_task = is_task_in_dag and ti.state == State.REMOVED<EOL>if should_restore_task:<EOL><INDENT>self.log.info("<STR_LIT>"<EOL>"<STR_LIT>".format(ti, dag))<EOL>Stats.incr("<STR_LIT>".format(dag.dag_id), <NUM_LIT:1>, <NUM_LIT:1>)<EOL>ti.state = State.NONE<EOL><DEDENT><DEDENT>for task in six.itervalues(dag.task_dict):<EOL><INDENT>if task.start_date > self.execution_date and not self.is_backfill:<EOL><INDENT>continue<EOL><DEDENT>if task.task_id not in task_ids:<EOL><INDENT>Stats.incr(<EOL>"<STR_LIT>".format(task.__class__.__name__),<EOL><NUM_LIT:1>, <NUM_LIT:1>)<EOL>ti = TaskInstance(task, self.execution_date)<EOL>session.add(ti)<EOL><DEDENT><DEDENT>session.commit()<EOL>

Verifies the DagRun by checking for removed tasks or tasks that are not in the
database yet. It will set state to removed or add the task if required.


@staticmethod<EOL><INDENT>def get_run(session, dag_id, execution_date):<DEDENT>

qry = session.query(DagRun).filter(<EOL>DagRun.dag_id == dag_id,<EOL>DagRun.external_trigger == False, <EOL>DagRun.execution_date == execution_date,<EOL>)<EOL>return qry.first()<EOL>

:param dag_id: DAG ID
:type dag_id: unicode
:param execution_date: execution date
:type execution_date: datetime
:return: DagRun corresponding to the given dag_id and execution date
    if one exists. None otherwise.
:rtype: airflow.models.DagRun


@classmethod<EOL><INDENT>def setdefault(cls, key, default, deserialize_json=False):<DEDENT>

obj = Variable.get(key, default_var=None,<EOL>deserialize_json=deserialize_json)<EOL>if obj is None:<EOL><INDENT>if default is not None:<EOL><INDENT>Variable.set(key, default, serialize_json=deserialize_json)<EOL>return default<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return obj<EOL><DEDENT>

Like a Python builtin dict object, setdefault returns the current value
for a key, and if it isn't there, stores the default value and returns it.

:param key: Dict key for this Variable
:type key: str
:param default: Default value to set and return if the variable
    isn't already in the DB
:type default: Mixed
:param deserialize_json: Store this as a JSON encoded value in the DB
    and un-encode it when retrieving a value
:return: Mixed


return len(self.dags)<EOL>

:return: the amount of dags contained in this dagbag


from airflow.models.dag import DagModel  <EOL>root_dag_id = dag_id<EOL>if dag_id in self.dags:<EOL><INDENT>dag = self.dags[dag_id]<EOL>if dag.is_subdag:<EOL><INDENT>root_dag_id = dag.parent_dag.dag_id<EOL><DEDENT><DEDENT>orm_dag = DagModel.get_current(root_dag_id)<EOL>if orm_dag and (<EOL>root_dag_id not in self.dags or<EOL>(<EOL>orm_dag.last_expired and<EOL>dag.last_loaded < orm_dag.last_expired<EOL>)<EOL>):<EOL><INDENT>found_dags = self.process_file(<EOL>filepath=orm_dag.fileloc, only_if_updated=False)<EOL>if found_dags and dag_id in [found_dag.dag_id for found_dag in found_dags]:<EOL><INDENT>return self.dags[dag_id]<EOL><DEDENT>elif dag_id in self.dags:<EOL><INDENT>del self.dags[dag_id]<EOL><DEDENT><DEDENT>return self.dags.get(dag_id)<EOL>

Gets the DAG out of the dictionary, and refreshes it if expired


def process_file(self, filepath, only_if_updated=True, safe_mode=True):

from airflow.models.dag import DAG  <EOL>found_dags = []<EOL>if filepath is None or not os.path.isfile(filepath):<EOL><INDENT>return found_dags<EOL><DEDENT>try:<EOL><INDENT>file_last_changed_on_disk = datetime.fromtimestamp(os.path.getmtime(filepath))<EOL>if only_if_updatedand filepath in self.file_last_changedand file_last_changed_on_disk == self.file_last_changed[filepath]:<EOL><INDENT>return found_dags<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>self.log.exception(e)<EOL>return found_dags<EOL><DEDENT>mods = []<EOL>is_zipfile = zipfile.is_zipfile(filepath)<EOL>if not is_zipfile:<EOL><INDENT>if safe_mode:<EOL><INDENT>with open(filepath, '<STR_LIT:rb>') as f:<EOL><INDENT>content = f.read()<EOL>if not all([s in content for s in (b'<STR_LIT>', b'<STR_LIT>')]):<EOL><INDENT>self.file_last_changed[filepath] = file_last_changed_on_disk<EOL>if not self.has_logged:<EOL><INDENT>self.has_logged = True<EOL>self.log.info(<EOL>"<STR_LIT>",<EOL>filepath)<EOL><DEDENT>return found_dags<EOL><DEDENT><DEDENT><DEDENT>self.log.debug("<STR_LIT>", filepath)<EOL>org_mod_name, _ = os.path.splitext(os.path.split(filepath)[-<NUM_LIT:1>])<EOL>mod_name = ('<STR_LIT>' +<EOL>hashlib.sha1(filepath.encode('<STR_LIT:utf-8>')).hexdigest() +<EOL>'<STR_LIT:_>' + org_mod_name)<EOL>if mod_name in sys.modules:<EOL><INDENT>del sys.modules[mod_name]<EOL><DEDENT>with timeout(configuration.conf.getint('<STR_LIT>', "<STR_LIT>")):<EOL><INDENT>try:<EOL><INDENT>m = imp.load_source(mod_name, filepath)<EOL>mods.append(m)<EOL><DEDENT>except Exception as e:<EOL><INDENT>self.log.exception("<STR_LIT>", filepath)<EOL>self.import_errors[filepath] = str(e)<EOL>self.file_last_changed[filepath] = file_last_changed_on_disk<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>zip_file = zipfile.ZipFile(filepath)<EOL>for mod in zip_file.infolist():<EOL><INDENT>head, _ = os.path.split(mod.filename)<EOL>mod_name, ext = os.path.splitext(mod.filename)<EOL>if not head and (ext == '<STR_LIT>' or ext == '<STR_LIT>'):<EOL><INDENT>if mod_name == '<STR_LIT>':<EOL><INDENT>self.log.warning("<STR_LIT>", ext, filepath)<EOL><DEDENT>if safe_mode:<EOL><INDENT>with zip_file.open(mod.filename) as zf:<EOL><INDENT>self.log.debug("<STR_LIT>", mod.filename, filepath)<EOL>content = zf.read()<EOL>if not all([s in content for s in (b'<STR_LIT>', b'<STR_LIT>')]):<EOL><INDENT>self.file_last_changed[filepath] = (<EOL>file_last_changed_on_disk)<EOL>if not self.has_logged:<EOL><INDENT>self.has_logged = True<EOL>self.log.info(<EOL>"<STR_LIT>",<EOL>filepath)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if mod_name in sys.modules:<EOL><INDENT>del sys.modules[mod_name]<EOL><DEDENT>try:<EOL><INDENT>sys.path.insert(<NUM_LIT:0>, filepath)<EOL>m = importlib.import_module(mod_name)<EOL>mods.append(m)<EOL><DEDENT>except Exception as e:<EOL><INDENT>self.log.exception("<STR_LIT>", filepath)<EOL>self.import_errors[filepath] = str(e)<EOL>self.file_last_changed[filepath] = file_last_changed_on_disk<EOL><DEDENT><DEDENT><DEDENT><DEDENT>for m in mods:<EOL><INDENT>for dag in list(m.__dict__.values()):<EOL><INDENT>if isinstance(dag, DAG):<EOL><INDENT>if not dag.full_filepath:<EOL><INDENT>dag.full_filepath = filepath<EOL>if dag.fileloc != filepath and not is_zipfile:<EOL><INDENT>dag.fileloc = filepath<EOL><DEDENT><DEDENT>try:<EOL><INDENT>dag.is_subdag = False<EOL>self.bag_dag(dag, parent_dag=dag, root_dag=dag)<EOL>if isinstance(dag._schedule_interval, six.string_types):<EOL><INDENT>croniter(dag._schedule_interval)<EOL><DEDENT>found_dags.append(dag)<EOL>found_dags += dag.subdags<EOL><DEDENT>except (CroniterBadCronError,<EOL>CroniterBadDateError,<EOL>CroniterNotAlphaError) as cron_e:<EOL><INDENT>self.log.exception("<STR_LIT>", dag.full_filepath)<EOL>self.import_errors[dag.full_filepath] ="<STR_LIT>" + str(cron_e)<EOL>self.file_last_changed[dag.full_filepath] =file_last_changed_on_disk<EOL><DEDENT>except AirflowDagCycleException as cycle_exception:<EOL><INDENT>self.log.exception("<STR_LIT>", dag.full_filepath)<EOL>self.import_errors[dag.full_filepath] = str(cycle_exception)<EOL>self.file_last_changed[dag.full_filepath] =file_last_changed_on_disk<EOL><DEDENT><DEDENT><DEDENT><DEDENT>self.file_last_changed[filepath] = file_last_changed_on_disk<EOL>return found_dags<EOL>

Given a path to a python module or zip file, this method imports
the module and look for dag objects within it.


@provide_session<EOL><INDENT>def kill_zombies(self, zombies, session=None):<DEDENT>

from airflow.models.taskinstance import TaskInstance  <EOL>for zombie in zombies:<EOL><INDENT>if zombie.dag_id in self.dags:<EOL><INDENT>dag = self.dags[zombie.dag_id]<EOL>if zombie.task_id in dag.task_ids:<EOL><INDENT>task = dag.get_task(zombie.task_id)<EOL>ti = TaskInstance(task, zombie.execution_date)<EOL>ti.start_date = zombie.start_date<EOL>ti.end_date = zombie.end_date<EOL>ti.try_number = zombie.try_number<EOL>ti.state = zombie.state<EOL>ti.test_mode = configuration.getboolean('<STR_LIT>', '<STR_LIT>')<EOL>ti.handle_failure("<STR_LIT>".format(ti),<EOL>ti.test_mode, ti.get_template_context())<EOL>self.log.info(<EOL>'<STR_LIT>', ti, ti.state)<EOL>Stats.incr('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>session.commit()<EOL>

Fail given zombie tasks, which are tasks that haven't
had a heartbeat for too long, in the current DagBag.

:param zombies: zombie task instances to kill.
:type zombies: airflow.utils.dag_processing.SimpleTaskInstance
:param session: DB session.
:type session: sqlalchemy.orm.session.Session


def bag_dag(self, dag, parent_dag, root_dag):

dag.test_cycle()  <EOL>dag.resolve_template_files()<EOL>dag.last_loaded = timezone.utcnow()<EOL>for task in dag.tasks:<EOL><INDENT>settings.policy(task)<EOL><DEDENT>subdags = dag.subdags<EOL>try:<EOL><INDENT>for subdag in subdags:<EOL><INDENT>subdag.full_filepath = dag.full_filepath<EOL>subdag.parent_dag = dag<EOL>subdag.is_subdag = True<EOL>self.bag_dag(subdag, parent_dag=dag, root_dag=root_dag)<EOL><DEDENT>self.dags[dag.dag_id] = dag<EOL>self.log.debug('<STR_LIT>', dag)<EOL><DEDENT>except AirflowDagCycleException as cycle_exception:<EOL><INDENT>self.log.exception('<STR_LIT>', dag.dag_id)<EOL>if dag == root_dag:<EOL><INDENT>for subdag in subdags:<EOL><INDENT>if subdag.dag_id in self.dags:<EOL><INDENT>del self.dags[subdag.dag_id]<EOL><DEDENT><DEDENT><DEDENT>raise cycle_exception<EOL><DEDENT>

Adds the DAG into the bag, recurses into sub dags.
Throws AirflowDagCycleException if a cycle is detected in this dag or its subdags


start_dttm = timezone.utcnow()<EOL>dag_folder = dag_folder or self.dag_folder<EOL>stats = []<EOL>FileLoadStat = namedtuple(<EOL>'<STR_LIT>', "<STR_LIT>")<EOL>dag_folder = correct_maybe_zipped(dag_folder)<EOL>for filepath in list_py_file_paths(dag_folder, safe_mode=safe_mode,<EOL>include_examples=include_examples):<EOL><INDENT>try:<EOL><INDENT>ts = timezone.utcnow()<EOL>found_dags = self.process_file(<EOL>filepath, only_if_updated=only_if_updated,<EOL>safe_mode=safe_mode)<EOL>td = timezone.utcnow() - ts<EOL>td = td.total_seconds() + (<EOL>float(td.microseconds) / <NUM_LIT>)<EOL>stats.append(FileLoadStat(<EOL>filepath.replace(dag_folder, '<STR_LIT>'),<EOL>td,<EOL>len(found_dags),<EOL>sum([len(dag.tasks) for dag in found_dags]),<EOL>str([dag.dag_id for dag in found_dags]),<EOL>))<EOL><DEDENT>except Exception as e:<EOL><INDENT>self.log.exception(e)<EOL><DEDENT><DEDENT>Stats.gauge(<EOL>'<STR_LIT>', (timezone.utcnow() - start_dttm).total_seconds(), <NUM_LIT:1>)<EOL>Stats.gauge(<EOL>'<STR_LIT>', len(self.dags), <NUM_LIT:1>)<EOL>Stats.gauge(<EOL>'<STR_LIT>', len(self.import_errors), <NUM_LIT:1>)<EOL>self.dagbag_stats = sorted(<EOL>stats, key=lambda x: x.duration, reverse=True)<EOL>

Given a file path or a folder, this method looks for python modules,
imports them and adds them to the dagbag collection.

Note that if a ``.airflowignore`` file is found while processing
the directory, it will behave much like a ``.gitignore``,
ignoring files that match any of the regex patterns specified
in the file.

**Note**: The patterns in .airflowignore are treated as
un-anchored regexes, not shell-like glob patterns.


report = textwrap.dedent("""<STR_LIT>""")<EOL>stats = self.dagbag_stats<EOL>return report.format(<EOL>dag_folder=self.dag_folder,<EOL>duration=sum([o.duration for o in stats]),<EOL>dag_num=sum([o.dag_num for o in stats]),<EOL>task_num=sum([o.task_num for o in stats]),<EOL>table=pprinttable(stats),<EOL>)<EOL>

Prints a report around DagBag loading stats


pass<EOL>

Will be called when an external kill command is received


try:<EOL><INDENT>with create_session() as session:<EOL><INDENT>job = session.query(BaseJob).filter_by(id=self.id).one()<EOL>make_transient(job)<EOL>session.commit()<EOL><DEDENT>if job.state == State.SHUTDOWN:<EOL><INDENT>self.kill()<EOL><DEDENT>is_unit_test = conf.getboolean('<STR_LIT>', '<STR_LIT>')<EOL>if not is_unit_test:<EOL><INDENT>sleep_for = <NUM_LIT:0><EOL>if job.latest_heartbeat:<EOL><INDENT>seconds_remaining = self.heartrate -(timezone.utcnow() - job.latest_heartbeat).total_seconds()<EOL>sleep_for = max(<NUM_LIT:0>, seconds_remaining)<EOL><DEDENT>sleep(sleep_for)<EOL><DEDENT>with create_session() as session:<EOL><INDENT>job = session.query(BaseJob).filter(BaseJob.id == self.id).first()<EOL>job.latest_heartbeat = timezone.utcnow()<EOL>session.merge(job)<EOL>session.commit()<EOL>self.heartbeat_callback(session=session)<EOL>self.log.debug('<STR_LIT>')<EOL><DEDENT><DEDENT>except OperationalError as e:<EOL><INDENT>self.log.error("<STR_LIT>", str(e))<EOL><DEDENT>

Heartbeats update the job's entry in the database with a timestamp
for the latest_heartbeat and allows for the job to be killed
externally. This allows at the system level to monitor what is
actually active.

For instance, an old heartbeat for SchedulerJob would mean something
is wrong.

This also allows for any job to be killed externally, regardless
of who is running it or on which machine it is running.

Note that if your heartbeat is set to 60 seconds and you call this
method after 10 seconds of processing since the last heartbeat, it
will sleep 50 seconds to complete the 60 seconds and keep a steady
heart rate. If you go over 60 seconds before calling it, it won't
sleep at all.


@provide_session<EOL><INDENT>def reset_state_for_orphaned_tasks(self, filter_by_dag_run=None, session=None):<DEDENT>

queued_tis = self.executor.queued_tasks<EOL>running_tis = self.executor.running<EOL>resettable_states = [State.SCHEDULED, State.QUEUED]<EOL>TI = models.TaskInstance<EOL>DR = models.DagRun<EOL>if filter_by_dag_run is None:<EOL><INDENT>resettable_tis = (<EOL>session<EOL>.query(TI)<EOL>.join(<EOL>DR,<EOL>and_(<EOL>TI.dag_id == DR.dag_id,<EOL>TI.execution_date == DR.execution_date))<EOL>.filter(<EOL>DR.state == State.RUNNING,<EOL>DR.run_id.notlike(BackfillJob.ID_PREFIX + '<STR_LIT:%>'),<EOL>TI.state.in_(resettable_states))).all()<EOL><DEDENT>else:<EOL><INDENT>resettable_tis = filter_by_dag_run.get_task_instances(state=resettable_states,<EOL>session=session)<EOL><DEDENT>tis_to_reset = []<EOL>for ti in resettable_tis:<EOL><INDENT>if ti.key not in queued_tis and ti.key not in running_tis:<EOL><INDENT>tis_to_reset.append(ti)<EOL><DEDENT><DEDENT>if len(tis_to_reset) == <NUM_LIT:0>:<EOL><INDENT>return []<EOL><DEDENT>def query(result, items):<EOL><INDENT>filter_for_tis = ([and_(TI.dag_id == ti.dag_id,<EOL>TI.task_id == ti.task_id,<EOL>TI.execution_date == ti.execution_date)<EOL>for ti in items])<EOL>reset_tis = (<EOL>session<EOL>.query(TI)<EOL>.filter(or_(*filter_for_tis), TI.state.in_(resettable_states))<EOL>.with_for_update()<EOL>.all())<EOL>for ti in reset_tis:<EOL><INDENT>ti.state = State.NONE<EOL>session.merge(ti)<EOL><DEDENT>return result + reset_tis<EOL><DEDENT>reset_tis = helpers.reduce_in_chunks(query,<EOL>tis_to_reset,<EOL>[],<EOL>self.max_tis_per_query)<EOL>task_instance_str = '<STR_LIT>'.join(<EOL>[repr(x) for x in reset_tis])<EOL>session.commit()<EOL>self.log.info(<EOL>"<STR_LIT>",<EOL>len(reset_tis), task_instance_str<EOL>)<EOL>return reset_tis<EOL>

This function checks if there are any tasks in the dagrun (or all)
that have a scheduled state but are not known by the
executor. If it finds those it will reset the state to None
so they will get picked up again.
The batch option is for performance reasons as the queries are made in
sequence.

:param filter_by_dag_run: the dag_run we want to process, None if all
:type filter_by_dag_run: airflow.models.DagRun
:return: the TIs reset (in expired SQLAlchemy state)
:rtype: list[airflow.models.TaskInstance]


def __init__(self, file_path, pickle_dags, dag_id_white_list, zombies):

self._file_path = file_path<EOL>self._result_queue = multiprocessing.Queue()<EOL>self._process = None<EOL>self._dag_id_white_list = dag_id_white_list<EOL>self._pickle_dags = pickle_dags<EOL>self._zombies = zombies<EOL>self._result = None<EOL>self._done = False<EOL>self._start_time = None<EOL>self._instance_id = DagFileProcessor.class_creation_counter<EOL>DagFileProcessor.class_creation_counter += <NUM_LIT:1><EOL>

:param file_path: a Python file containing Airflow DAG definitions
:type file_path: unicode
:param pickle_dags: whether to serialize the DAG objects to the DB
:type pickle_dags: bool
:param dag_id_whitelist: If specified, only look at these DAG ID's
:type dag_id_whitelist: list[unicode]
:param zombies: zombie task instances to kill
:type zombies: list[airflow.utils.dag_processing.SimpleTaskInstance]


def helper():<EOL><INDENT>log = logging.getLogger("<STR_LIT>")<EOL>stdout = StreamLogWriter(log, logging.INFO)<EOL>stderr = StreamLogWriter(log, logging.WARN)<EOL>set_context(log, file_path)<EOL>try:<EOL><INDENT>sys.stdout = stdout<EOL>sys.stderr = stderr<EOL>settings.configure_orm()<EOL>threading.current_thread().name = thread_name<EOL>start_time = time.time()<EOL>log.info("<STR_LIT>",<EOL>os.getpid(), file_path)<EOL>scheduler_job = SchedulerJob(dag_ids=dag_id_white_list, log=log)<EOL>result = scheduler_job.process_file(file_path,<EOL>zombies,<EOL>pickle_dags)<EOL>result_queue.put(result)<EOL>end_time = time.time()<EOL>log.info(<EOL>"<STR_LIT>", file_path, end_time - start_time<EOL>)<EOL><DEDENT>except Exception:<EOL><INDENT>log.exception("<STR_LIT>")<EOL>raise<EOL><DEDENT>finally:<EOL><INDENT>sys.stdout = sys.__stdout__<EOL>sys.stderr = sys.__stderr__<EOL>settings.dispose_orm()<EOL><DEDENT><DEDENT>p = multiprocessing.Process(target=helper,<EOL>args=(),<EOL>name="<STR_LIT>".format(thread_name))<EOL>p.start()<EOL>return p<EOL>

Launch a process to process the given file.

:param result_queue: the queue to use for passing back the result
:type result_queue: multiprocessing.Queue
:param file_path: the file to process
:type file_path: unicode
:param pickle_dags: whether to pickle the DAGs found in the file and
    save them to the DB
:type pickle_dags: bool
:param dag_id_white_list: if specified, only examine DAG ID's that are
    in this list
:type dag_id_white_list: list[unicode]
:param thread_name: the name to use for the process that is launched
:type thread_name: unicode
:return: the process that was launched
:rtype: multiprocessing.Process
:param zombies: zombie task instances to kill
:type zombies: list[airflow.utils.dag_processing.SimpleTaskInstance]


Launch the process and start processing the DAG.


if self._process is None:<EOL><INDENT>raise AirflowException("<STR_LIT>")<EOL><DEDENT>self._result_queue = None<EOL>self._process.terminate()<EOL>self._process.join(<NUM_LIT:5>)<EOL>if sigkill and self._process.is_alive():<EOL><INDENT>self.log.warning("<STR_LIT>", self._process.pid)<EOL>os.kill(self._process.pid, signal.SIGKILL)<EOL><DEDENT>

Terminate (and then kill) the process launched to process the file.

:param sigkill: whether to issue a SIGKILL if SIGTERM doesn't work.
:type sigkill: bool


if self._process is None:<EOL><INDENT>raise AirflowException("<STR_LIT>")<EOL><DEDENT>return self._process.pid<EOL>

:return: the PID of the process launched to process the given file
:rtype: int


if not self._done:<EOL><INDENT>raise AirflowException("<STR_LIT>")<EOL><DEDENT>return self._process.exitcode<EOL>

After the process is finished, this can be called to get the return code

:return: the exit code of the process
:rtype: int


if self._process is None:<EOL><INDENT>raise AirflowException("<STR_LIT>")<EOL><DEDENT>if self._done:<EOL><INDENT>return True<EOL><DEDENT>if self._result_queue and not self._result_queue.empty():<EOL><INDENT>self._result = self._result_queue.get_nowait()<EOL>self._done = True<EOL>self.log.debug("<STR_LIT>", self._process)<EOL>self._process.join()<EOL>return True<EOL><DEDENT>if self._result_queue and not self._process.is_alive():<EOL><INDENT>self._done = True<EOL>if not self._result_queue.empty():<EOL><INDENT>self._result = self._result_queue.get_nowait()<EOL><DEDENT>self.log.debug("<STR_LIT>", self._process)<EOL>self._process.join()<EOL>return True<EOL><DEDENT>return False<EOL>

Check if the process launched to process this file is done.

:return: whether the process is finished running
:rtype: bool


if not self.done:<EOL><INDENT>raise AirflowException("<STR_LIT>")<EOL><DEDENT>return self._result<EOL>

:return: result of running SchedulerJob.process_file()
:rtype: airflow.utils.dag_processing.SimpleDag


if self._start_time is None:<EOL><INDENT>raise AirflowException("<STR_LIT>")<EOL><DEDENT>return self._start_time<EOL>

:return: when this started to process the file
:rtype: datetime


<EOL>self.dag_id = dag_id<EOL>self.dag_ids = [dag_id] if dag_id else []<EOL>if dag_ids:<EOL><INDENT>self.dag_ids.extend(dag_ids)<EOL><DEDENT>self.subdir = subdir<EOL>self.num_runs = num_runs<EOL>self._processor_poll_interval = processor_poll_interval<EOL>self.do_pickle = do_pickle<EOL>super().__init__(*args, **kwargs)<EOL>self.heartrate = conf.getint('<STR_LIT>', '<STR_LIT>')<EOL>self.max_threads = conf.getint('<STR_LIT>', '<STR_LIT>')<EOL>if log:<EOL><INDENT>self._log = log<EOL><DEDENT>self.using_sqlite = False<EOL>if '<STR_LIT>' in conf.get('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>self.using_sqlite = True<EOL><DEDENT>self.max_tis_per_query = conf.getint('<STR_LIT>', '<STR_LIT>')<EOL>self.processor_agent = None<EOL>self._last_loop = False<EOL>signal.signal(signal.SIGINT, self._exit_gracefully)<EOL>signal.signal(signal.SIGTERM, self._exit_gracefully)<EOL>

:param dag_id: if specified, only schedule tasks with this DAG ID
:type dag_id: unicode
:param dag_ids: if specified, only schedule tasks with these DAG IDs
:type dag_ids: list[unicode]
:param subdir: directory containing Python files with Airflow DAG
    definitions, or a specific path to a file
:type subdir: unicode
:param num_runs: The number of times to try to schedule each DAG file.
    -1 for unlimited times.
:type num_runs: int
:param processor_poll_interval: The number of seconds to wait between
    polls of running processors
:type processor_poll_interval: int
:param do_pickle: once a DAG object is obtained by executing the Python
    file, whether to serialize the DAG object to the DB
:type do_pickle: bool


def _exit_gracefully(self, signum, frame):

self.log.info("<STR_LIT>", signum)<EOL>if self.processor_agent:<EOL><INDENT>self.processor_agent.end()<EOL><DEDENT>sys.exit(os.EX_OK)<EOL>

Helper method to clean up processor_agent to avoid leaving orphan processes.


@provide_session<EOL><INDENT>def manage_slas(self, dag, session=None):<DEDENT>

if not any([isinstance(ti.sla, timedelta) for ti in dag.tasks]):<EOL><INDENT>self.log.info("<STR_LIT>", dag)<EOL>return<EOL><DEDENT>TI = models.TaskInstance<EOL>sq = (<EOL>session<EOL>.query(<EOL>TI.task_id,<EOL>func.max(TI.execution_date).label('<STR_LIT>'))<EOL>.with_hint(TI, '<STR_LIT>', dialect_name='<STR_LIT>')<EOL>.filter(TI.dag_id == dag.dag_id)<EOL>.filter(or_(<EOL>TI.state == State.SUCCESS,<EOL>TI.state == State.SKIPPED))<EOL>.filter(TI.task_id.in_(dag.task_ids))<EOL>.group_by(TI.task_id).subquery('<STR_LIT>')<EOL>)<EOL>max_tis = session.query(TI).filter(<EOL>TI.dag_id == dag.dag_id,<EOL>TI.task_id == sq.c.task_id,<EOL>TI.execution_date == sq.c.max_ti,<EOL>).all()<EOL>ts = timezone.utcnow()<EOL>for ti in max_tis:<EOL><INDENT>task = dag.get_task(ti.task_id)<EOL>dttm = ti.execution_date<EOL>if isinstance(task.sla, timedelta):<EOL><INDENT>dttm = dag.following_schedule(dttm)<EOL>while dttm < timezone.utcnow():<EOL><INDENT>following_schedule = dag.following_schedule(dttm)<EOL>if following_schedule + task.sla < timezone.utcnow():<EOL><INDENT>session.merge(SlaMiss(<EOL>task_id=ti.task_id,<EOL>dag_id=ti.dag_id,<EOL>execution_date=dttm,<EOL>timestamp=ts))<EOL><DEDENT>dttm = dag.following_schedule(dttm)<EOL><DEDENT><DEDENT><DEDENT>session.commit()<EOL>slas = (<EOL>session<EOL>.query(SlaMiss)<EOL>.filter(SlaMiss.notification_sent == False, SlaMiss.dag_id == dag.dag_id)  <EOL>.all()<EOL>)<EOL>if slas:<EOL><INDENT>sla_dates = [sla.execution_date for sla in slas]<EOL>qry = (<EOL>session<EOL>.query(TI)<EOL>.filter(<EOL>TI.state != State.SUCCESS,<EOL>TI.execution_date.in_(sla_dates),<EOL>TI.dag_id == dag.dag_id<EOL>).all()<EOL>)<EOL>blocking_tis = []<EOL>for ti in qry:<EOL><INDENT>if ti.task_id in dag.task_ids:<EOL><INDENT>ti.task = dag.get_task(ti.task_id)<EOL>blocking_tis.append(ti)<EOL><DEDENT>else:<EOL><INDENT>session.delete(ti)<EOL>session.commit()<EOL><DEDENT><DEDENT>task_list = "<STR_LIT:\n>".join([<EOL>sla.task_id + '<STR_LIT>' + sla.execution_date.isoformat()<EOL>for sla in slas])<EOL>blocking_task_list = "<STR_LIT:\n>".join([<EOL>ti.task_id + '<STR_LIT>' + ti.execution_date.isoformat()<EOL>for ti in blocking_tis])<EOL>email_sent = False<EOL>notification_sent = False<EOL>if dag.sla_miss_callback:<EOL><INDENT>self.log.info('<STR_LIT>')<EOL>try:<EOL><INDENT>dag.sla_miss_callback(dag, task_list, blocking_task_list, slas,<EOL>blocking_tis)<EOL>notification_sent = True<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.exception("<STR_LIT>",<EOL>dag.dag_id)<EOL><DEDENT><DEDENT>email_content = """<STR_LIT>""".format(task_list=task_list, blocking_task_list=blocking_task_list,<EOL>bug=asciiart.bug)<EOL>emails = set()<EOL>for task in dag.tasks:<EOL><INDENT>if task.email:<EOL><INDENT>if isinstance(task.email, basestring):<EOL><INDENT>emails |= set(get_email_address_list(task.email))<EOL><DEDENT>elif isinstance(task.email, (list, tuple)):<EOL><INDENT>emails |= set(task.email)<EOL><DEDENT><DEDENT><DEDENT>if emails:<EOL><INDENT>try:<EOL><INDENT>send_email(<EOL>emails,<EOL>"<STR_LIT>" + dag.dag_id,<EOL>email_content)<EOL>email_sent = True<EOL>notification_sent = True<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.exception("<STR_LIT>"<EOL>"<STR_LIT>", dag.dag_id)<EOL><DEDENT><DEDENT>if notification_sent:<EOL><INDENT>for sla in slas:<EOL><INDENT>if email_sent:<EOL><INDENT>sla.email_sent = True<EOL><DEDENT>sla.notification_sent = True<EOL>session.merge(sla)<EOL><DEDENT><DEDENT>session.commit()<EOL><DEDENT>

Finding all tasks that have SLAs defined, and sending alert emails
where needed. New SLA misses are also recorded in the database.

Where assuming that the scheduler runs often, so we only check for
tasks that should have succeeded in the past hour.


<EOL>for dagbag_file in dagbag.file_last_changed:<EOL><INDENT>session.query(errors.ImportError).filter(<EOL>errors.ImportError.filename == dagbag_file<EOL>).delete()<EOL><DEDENT>for filename, stacktrace in six.iteritems(dagbag.import_errors):<EOL><INDENT>session.add(errors.ImportError(<EOL>filename=filename,<EOL>stacktrace=stacktrace))<EOL><DEDENT>session.commit()<EOL>

For the DAGs in the given DagBag, record any associated import errors and clears
errors for files that no longer have them. These are usually displayed through the
Airflow UI so that users know that there are issues parsing DAGs.

:param session: session for ORM operations
:type session: sqlalchemy.orm.session.Session
:param dagbag: DagBag containing DAGs with import errors
:type dagbag: airflow.models.DagBag


@provide_session<EOL><INDENT>def create_dag_run(self, dag, session=None):<DEDENT>

if dag.schedule_interval and conf.getboolean('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>active_runs = DagRun.find(<EOL>dag_id=dag.dag_id,<EOL>state=State.RUNNING,<EOL>external_trigger=False,<EOL>session=session<EOL>)<EOL>if len(active_runs) >= dag.max_active_runs and not dag.dagrun_timeout:<EOL><INDENT>return<EOL><DEDENT>timedout_runs = <NUM_LIT:0><EOL>for dr in active_runs:<EOL><INDENT>if (<EOL>dr.start_date and dag.dagrun_timeout and<EOL>dr.start_date < timezone.utcnow() - dag.dagrun_timeout):<EOL><INDENT>dr.state = State.FAILED<EOL>dr.end_date = timezone.utcnow()<EOL>dag.handle_callback(dr, success=False, reason='<STR_LIT>',<EOL>session=session)<EOL>timedout_runs += <NUM_LIT:1><EOL><DEDENT><DEDENT>session.commit()<EOL>if len(active_runs) - timedout_runs >= dag.max_active_runs:<EOL><INDENT>return<EOL><DEDENT>qry = (<EOL>session.query(func.max(DagRun.execution_date))<EOL>.filter_by(dag_id=dag.dag_id)<EOL>.filter(or_(<EOL>DagRun.external_trigger == False,  <EOL>DagRun.run_id.like(DagRun.ID_PREFIX + '<STR_LIT:%>')<EOL>))<EOL>)<EOL>last_scheduled_run = qry.scalar()<EOL>if dag.schedule_interval == '<STR_LIT>' and last_scheduled_run:<EOL><INDENT>return None<EOL><DEDENT>if not (dag.catchup or dag.schedule_interval == '<STR_LIT>'):<EOL><INDENT>now = timezone.utcnow()<EOL>next_start = dag.following_schedule(now)<EOL>last_start = dag.previous_schedule(now)<EOL>if next_start <= now:<EOL><INDENT>new_start = last_start<EOL><DEDENT>else:<EOL><INDENT>new_start = dag.previous_schedule(last_start)<EOL><DEDENT>if dag.start_date:<EOL><INDENT>if new_start >= dag.start_date:<EOL><INDENT>dag.start_date = new_start<EOL><DEDENT><DEDENT>else:<EOL><INDENT>dag.start_date = new_start<EOL><DEDENT><DEDENT>next_run_date = None<EOL>if not last_scheduled_run:<EOL><INDENT>task_start_dates = [t.start_date for t in dag.tasks]<EOL>if task_start_dates:<EOL><INDENT>next_run_date = dag.normalize_schedule(min(task_start_dates))<EOL>self.log.debug(<EOL>"<STR_LIT>",<EOL>next_run_date<EOL>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>next_run_date = dag.following_schedule(last_scheduled_run)<EOL><DEDENT>last_run = dag.get_last_dagrun(session=session)<EOL>if last_run and next_run_date:<EOL><INDENT>while next_run_date <= last_run.execution_date:<EOL><INDENT>next_run_date = dag.following_schedule(next_run_date)<EOL><DEDENT><DEDENT>if dag.start_date:<EOL><INDENT>next_run_date = (dag.start_date if not next_run_date<EOL>else max(next_run_date, dag.start_date))<EOL>if next_run_date == dag.start_date:<EOL><INDENT>next_run_date = dag.normalize_schedule(dag.start_date)<EOL><DEDENT>self.log.debug(<EOL>"<STR_LIT>",<EOL>dag.start_date, next_run_date<EOL>)<EOL><DEDENT>if not next_run_date or next_run_date > timezone.utcnow():<EOL><INDENT>return<EOL><DEDENT>if dag.schedule_interval == '<STR_LIT>':<EOL><INDENT>period_end = next_run_date<EOL><DEDENT>elif next_run_date:<EOL><INDENT>period_end = dag.following_schedule(next_run_date)<EOL><DEDENT>if next_run_date and dag.end_date and next_run_date > dag.end_date:<EOL><INDENT>return<EOL><DEDENT>min_task_end_date = []<EOL>task_end_dates = [t.end_date for t in dag.tasks if t.end_date]<EOL>if task_end_dates:<EOL><INDENT>min_task_end_date = min(task_end_dates)<EOL><DEDENT>if next_run_date and min_task_end_date and next_run_date > min_task_end_date:<EOL><INDENT>return<EOL><DEDENT>if next_run_date and period_end and period_end <= timezone.utcnow():<EOL><INDENT>next_run = dag.create_dagrun(<EOL>run_id=DagRun.ID_PREFIX + next_run_date.isoformat(),<EOL>execution_date=next_run_date,<EOL>start_date=timezone.utcnow(),<EOL>state=State.RUNNING,<EOL>external_trigger=False<EOL>)<EOL>return next_run<EOL><DEDENT><DEDENT>

This method checks whether a new DagRun needs to be created
for a DAG based on scheduling interval.
Returns DagRun if one is scheduled. Otherwise returns None.


@provide_session<EOL><INDENT>def _process_task_instances(self, dag, queue, session=None):<DEDENT>

<EOL>dag_runs = DagRun.find(dag_id=dag.dag_id, state=State.RUNNING, session=session)<EOL>active_dag_runs = []<EOL>for run in dag_runs:<EOL><INDENT>self.log.info("<STR_LIT>", run)<EOL>if run.execution_date > timezone.utcnow():<EOL><INDENT>self.log.error(<EOL>"<STR_LIT>",<EOL>run.execution_date<EOL>)<EOL>continue<EOL><DEDENT>if len(active_dag_runs) >= dag.max_active_runs:<EOL><INDENT>self.log.info("<STR_LIT>")<EOL>break<EOL><DEDENT>if run.is_backfill:<EOL><INDENT>continue<EOL><DEDENT>run.dag = dag<EOL>run.verify_integrity(session=session)<EOL>run.update_state(session=session)<EOL>if run.state == State.RUNNING:<EOL><INDENT>make_transient(run)<EOL>active_dag_runs.append(run)<EOL><DEDENT><DEDENT>for run in active_dag_runs:<EOL><INDENT>self.log.debug("<STR_LIT>", run)<EOL>tis = run.get_task_instances(state=(State.NONE,<EOL>State.UP_FOR_RETRY,<EOL>State.UP_FOR_RESCHEDULE))<EOL>for ti in tis:<EOL><INDENT>task = dag.get_task(ti.task_id)<EOL>ti.task = task<EOL>if ti.are_dependencies_met(<EOL>dep_context=DepContext(flag_upstream_failed=True),<EOL>session=session):<EOL><INDENT>self.log.debug('<STR_LIT>', ti)<EOL>queue.append(ti.key)<EOL><DEDENT><DEDENT><DEDENT>

This method schedules the tasks for a single DAG by looking at the
active DAG runs and adding task instances that should run to the
queue.


tis_changed = <NUM_LIT:0><EOL>query = session.query(models.TaskInstance).outerjoin(models.DagRun, and_(<EOL>models.TaskInstance.dag_id == models.DagRun.dag_id,<EOL>models.TaskInstance.execution_date == models.DagRun.execution_date)).filter(models.TaskInstance.dag_id.in_(simple_dag_bag.dag_ids)).filter(models.TaskInstance.state.in_(old_states)).filter(or_(<EOL>models.DagRun.state != State.RUNNING,<EOL>models.DagRun.state.is_(None)))<EOL>if self.using_sqlite:<EOL><INDENT>tis_to_change = query.with_for_update().all()<EOL>for ti in tis_to_change:<EOL><INDENT>ti.set_state(new_state, session=session)<EOL>tis_changed += <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>subq = query.subquery()<EOL>tis_changed = session.query(models.TaskInstance).filter(and_(<EOL>models.TaskInstance.dag_id == subq.c.dag_id,<EOL>models.TaskInstance.task_id == subq.c.task_id,<EOL>models.TaskInstance.execution_date ==<EOL>subq.c.execution_date)).update({models.TaskInstance.state: new_state},<EOL>synchronize_session=False)<EOL>session.commit()<EOL><DEDENT>if tis_changed > <NUM_LIT:0>:<EOL><INDENT>self.log.warning(<EOL>"<STR_LIT>",<EOL>tis_changed, new_state<EOL>)<EOL><DEDENT>

For all DAG IDs in the SimpleDagBag, look for task instances in the
old_states and set them to new_state if the corresponding DagRun
does not exist or exists but is not in the running state. This
normally should not happen, but it can if the state of DagRuns are
changed manually.

:param old_states: examine TaskInstances in this state
:type old_state: list[airflow.utils.state.State]
:param new_state: set TaskInstances to this state
:type new_state: airflow.utils.state.State
:param simple_dag_bag: TaskInstances associated with DAGs in the
    simple_dag_bag and with states in the old_state will be examined
:type simple_dag_bag: airflow.utils.dag_processing.SimpleDagBag


@provide_session<EOL><INDENT>def __get_concurrency_maps(self, states, session=None):<DEDENT>

TI = models.TaskInstance<EOL>ti_concurrency_query = (<EOL>session<EOL>.query(TI.task_id, TI.dag_id, func.count('<STR_LIT:*>'))<EOL>.filter(TI.state.in_(states))<EOL>.group_by(TI.task_id, TI.dag_id)<EOL>).all()<EOL>dag_map = defaultdict(int)<EOL>task_map = defaultdict(int)<EOL>for result in ti_concurrency_query:<EOL><INDENT>task_id, dag_id, count = result<EOL>dag_map[dag_id] += count<EOL>task_map[(dag_id, task_id)] = count<EOL><DEDENT>return dag_map, task_map<EOL>

Get the concurrency maps.

:param states: List of states to query for
:type states: list[airflow.utils.state.State]
:return: A map from (dag_id, task_id) to # of task instances and
 a map from (dag_id, task_id) to # of task instances in the given state list
:rtype: dict[tuple[str, str], int]


@provide_session<EOL><INDENT>def _find_executable_task_instances(self, simple_dag_bag, states, session=None):<DEDENT>

executable_tis = []<EOL>TI = models.TaskInstance<EOL>DR = models.DagRun<EOL>DM = models.DagModel<EOL>ti_query = (<EOL>session<EOL>.query(TI)<EOL>.filter(TI.dag_id.in_(simple_dag_bag.dag_ids))<EOL>.outerjoin(<EOL>DR,<EOL>and_(DR.dag_id == TI.dag_id, DR.execution_date == TI.execution_date)<EOL>)<EOL>.filter(or_(DR.run_id == None,  <EOL>not_(DR.run_id.like(BackfillJob.ID_PREFIX + '<STR_LIT:%>'))))<EOL>.outerjoin(DM, DM.dag_id == TI.dag_id)<EOL>.filter(or_(DM.dag_id == None,  <EOL>not_(DM.is_paused)))<EOL>)<EOL>if None in states:<EOL><INDENT>ti_query = ti_query.filter(<EOL>or_(TI.state == None, TI.state.in_(states))  <EOL>)<EOL><DEDENT>else:<EOL><INDENT>ti_query = ti_query.filter(TI.state.in_(states))<EOL><DEDENT>task_instances_to_examine = ti_query.all()<EOL>if len(task_instances_to_examine) == <NUM_LIT:0>:<EOL><INDENT>self.log.debug("<STR_LIT>")<EOL>return executable_tis<EOL><DEDENT>task_instance_str = "<STR_LIT>".join(<EOL>[repr(x) for x in task_instances_to_examine])<EOL>self.log.info(<EOL>"<STR_LIT>", len(task_instances_to_examine),<EOL>task_instance_str<EOL>)<EOL>pools = {p.pool: p for p in session.query(models.Pool).all()}<EOL>pool_to_task_instances = defaultdict(list)<EOL>for task_instance in task_instances_to_examine:<EOL><INDENT>pool_to_task_instances[task_instance.pool].append(task_instance)<EOL><DEDENT>states_to_count_as_running = [State.RUNNING, State.QUEUED]<EOL>dag_concurrency_map, task_concurrency_map = self.__get_concurrency_maps(<EOL>states=states_to_count_as_running, session=session)<EOL>for pool, task_instances in pool_to_task_instances.items():<EOL><INDENT>pool_name = pool<EOL>if not pool:<EOL><INDENT>open_slots = models.Pool.default_pool_open_slots()<EOL>pool_name = models.Pool.default_pool_name<EOL><DEDENT>else:<EOL><INDENT>if pool not in pools:<EOL><INDENT>self.log.warning(<EOL>"<STR_LIT>",<EOL>pool<EOL>)<EOL>open_slots = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>open_slots = pools[pool].open_slots(session=session)<EOL><DEDENT><DEDENT>num_ready = len(task_instances)<EOL>self.log.info(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>pool, open_slots, num_ready<EOL>)<EOL>priority_sorted_task_instances = sorted(<EOL>task_instances, key=lambda ti: (-ti.priority_weight, ti.execution_date))<EOL>num_starving_tasks = <NUM_LIT:0><EOL>for current_index, task_instance in enumerate(priority_sorted_task_instances):<EOL><INDENT>if open_slots <= <NUM_LIT:0>:<EOL><INDENT>self.log.info(<EOL>"<STR_LIT>",<EOL>open_slots, pool<EOL>)<EOL>num_starving_tasks = len(priority_sorted_task_instances) - current_index<EOL>break<EOL><DEDENT>dag_id = task_instance.dag_id<EOL>simple_dag = simple_dag_bag.get_dag(dag_id)<EOL>current_dag_concurrency = dag_concurrency_map[dag_id]<EOL>dag_concurrency_limit = simple_dag_bag.get_dag(dag_id).concurrency<EOL>self.log.info(<EOL>"<STR_LIT>",<EOL>dag_id, current_dag_concurrency, dag_concurrency_limit<EOL>)<EOL>if current_dag_concurrency >= dag_concurrency_limit:<EOL><INDENT>self.log.info(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>task_instance, dag_id, dag_concurrency_limit<EOL>)<EOL>continue<EOL><DEDENT>task_concurrency_limit = simple_dag.get_task_special_arg(<EOL>task_instance.task_id,<EOL>'<STR_LIT>')<EOL>if task_concurrency_limit is not None:<EOL><INDENT>current_task_concurrency = task_concurrency_map[<EOL>(task_instance.dag_id, task_instance.task_id)<EOL>]<EOL>if current_task_concurrency >= task_concurrency_limit:<EOL><INDENT>self.log.info("<STR_LIT>"<EOL>"<STR_LIT>", task_instance)<EOL>continue<EOL><DEDENT><DEDENT>if self.executor.has_task(task_instance):<EOL><INDENT>self.log.debug(<EOL>"<STR_LIT>",<EOL>task_instance.key<EOL>)<EOL>continue<EOL><DEDENT>executable_tis.append(task_instance)<EOL>open_slots -= <NUM_LIT:1><EOL>dag_concurrency_map[dag_id] += <NUM_LIT:1><EOL>task_concurrency_map[(task_instance.dag_id, task_instance.task_id)] += <NUM_LIT:1><EOL><DEDENT>Stats.gauge('<STR_LIT>'.format(pool_name=pool_name),<EOL>num_starving_tasks)<EOL><DEDENT>task_instance_str = "<STR_LIT>".join(<EOL>[repr(x) for x in executable_tis])<EOL>self.log.info(<EOL>"<STR_LIT>", task_instance_str)<EOL>for ti in executable_tis:<EOL><INDENT>copy_dag_id = ti.dag_id<EOL>copy_execution_date = ti.execution_date<EOL>copy_task_id = ti.task_id<EOL>make_transient(ti)<EOL>ti.dag_id = copy_dag_id<EOL>ti.execution_date = copy_execution_date<EOL>ti.task_id = copy_task_id<EOL><DEDENT>return executable_tis<EOL>

Finds TIs that are ready for execution with respect to pool limits,
dag concurrency, executor state, and priority.

:param simple_dag_bag: TaskInstances associated with DAGs in the
    simple_dag_bag will be fetched from the DB and executed
:type simple_dag_bag: airflow.utils.dag_processing.SimpleDagBag
:param executor: the executor that runs task instances
:type executor: BaseExecutor
:param states: Execute TaskInstances in these states
:type states: tuple[airflow.utils.state.State]
:return: list[airflow.models.TaskInstance]


if len(task_instances) == <NUM_LIT:0>:<EOL><INDENT>session.commit()<EOL>return []<EOL><DEDENT>TI = models.TaskInstance<EOL>filter_for_ti_state_change = (<EOL>[and_(<EOL>TI.dag_id == ti.dag_id,<EOL>TI.task_id == ti.task_id,<EOL>TI.execution_date == ti.execution_date)<EOL>for ti in task_instances])<EOL>ti_query = (<EOL>session<EOL>.query(TI)<EOL>.filter(or_(*filter_for_ti_state_change)))<EOL>if None in acceptable_states:<EOL><INDENT>ti_query = ti_query.filter(<EOL>or_(TI.state == None, TI.state.in_(acceptable_states))  <EOL>)<EOL><DEDENT>else:<EOL><INDENT>ti_query = ti_query.filter(TI.state.in_(acceptable_states))<EOL><DEDENT>tis_to_set_to_queued = (<EOL>ti_query<EOL>.with_for_update()<EOL>.all())<EOL>if len(tis_to_set_to_queued) == <NUM_LIT:0>:<EOL><INDENT>self.log.info("<STR_LIT>")<EOL>session.commit()<EOL>return []<EOL><DEDENT>for task_instance in tis_to_set_to_queued:<EOL><INDENT>task_instance.state = State.QUEUED<EOL>task_instance.queued_dttm = (timezone.utcnow()<EOL>if not task_instance.queued_dttm<EOL>else task_instance.queued_dttm)<EOL>session.merge(task_instance)<EOL><DEDENT>simple_task_instances = [SimpleTaskInstance(ti) for ti in<EOL>tis_to_set_to_queued]<EOL>task_instance_str = "<STR_LIT>".join(<EOL>[repr(x) for x in tis_to_set_to_queued])<EOL>session.commit()<EOL>self.log.info("<STR_LIT>",<EOL>len(tis_to_set_to_queued), task_instance_str)<EOL>return simple_task_instances<EOL>

Changes the state of task instances in the list with one of the given states
to QUEUED atomically, and returns the TIs changed in SimpleTaskInstance format.

:param task_instances: TaskInstances to change the state of
:type task_instances: list[airflow.models.TaskInstance]
:param acceptable_states: Filters the TaskInstances updated to be in these states
:type acceptable_states: Iterable[State]
:rtype: list[airflow.utils.dag_processing.SimpleTaskInstance]


TI = models.TaskInstance<EOL>for simple_task_instance in simple_task_instances:<EOL><INDENT>simple_dag = simple_dag_bag.get_dag(simple_task_instance.dag_id)<EOL>command = TI.generate_command(<EOL>simple_task_instance.dag_id,<EOL>simple_task_instance.task_id,<EOL>simple_task_instance.execution_date,<EOL>local=True,<EOL>mark_success=False,<EOL>ignore_all_deps=False,<EOL>ignore_depends_on_past=False,<EOL>ignore_task_deps=False,<EOL>ignore_ti_state=False,<EOL>pool=simple_task_instance.pool,<EOL>file_path=simple_dag.full_filepath,<EOL>pickle_id=simple_dag.pickle_id)<EOL>priority = simple_task_instance.priority_weight<EOL>queue = simple_task_instance.queue<EOL>self.log.info(<EOL>"<STR_LIT>",<EOL>simple_task_instance.key, priority, queue<EOL>)<EOL>self.executor.queue_command(<EOL>simple_task_instance,<EOL>command,<EOL>priority=priority,<EOL>queue=queue)<EOL><DEDENT>

Takes task_instances, which should have been set to queued, and enqueues them
with the executor.

:param simple_task_instances: TaskInstances to enqueue
:type simple_task_instances: list[SimpleTaskInstance]
:param simple_dag_bag: Should contains all of the task_instances' dags
:type simple_dag_bag: airflow.utils.dag_processing.SimpleDagBag


executable_tis = self._find_executable_task_instances(simple_dag_bag, states,<EOL>session=session)<EOL>def query(result, items):<EOL><INDENT>simple_tis_with_state_changed =self._change_state_for_executable_task_instances(items,<EOL>states,<EOL>session=session)<EOL>self._enqueue_task_instances_with_queued_state(<EOL>simple_dag_bag,<EOL>simple_tis_with_state_changed)<EOL>session.commit()<EOL>return result + len(simple_tis_with_state_changed)<EOL><DEDENT>return helpers.reduce_in_chunks(query, executable_tis, <NUM_LIT:0>, self.max_tis_per_query)<EOL>

Attempts to execute TaskInstances that should be executed by the scheduler.

There are three steps:
1. Pick TIs by priority with the constraint that they are in the expected states
and that we do exceed max_active_runs or pool limits.
2. Change the state for the TIs above atomically.
3. Enqueue the TIs in the executor.

:param simple_dag_bag: TaskInstances associated with DAGs in the
    simple_dag_bag will be fetched from the DB and executed
:type simple_dag_bag: airflow.utils.dag_processing.SimpleDagBag
:param states: Execute TaskInstances in these states
:type states: tuple[airflow.utils.state.State]
:return: Number of task instance with state changed.


if self.executor.queued_tasks:<EOL><INDENT>TI = models.TaskInstance<EOL>filter_for_ti_state_change = (<EOL>[and_(<EOL>TI.dag_id == dag_id,<EOL>TI.task_id == task_id,<EOL>TI.execution_date == execution_date,<EOL>TI._try_number == try_number - <NUM_LIT:1>,<EOL>TI.state == State.QUEUED)<EOL>for dag_id, task_id, execution_date, try_number<EOL>in self.executor.queued_tasks.keys()])<EOL>ti_query = (session.query(TI)<EOL>.filter(or_(*filter_for_ti_state_change)))<EOL>tis_to_set_to_scheduled = (ti_query<EOL>.with_for_update()<EOL>.all())<EOL>if len(tis_to_set_to_scheduled) == <NUM_LIT:0>:<EOL><INDENT>session.commit()<EOL>return<EOL><DEDENT>for task_instance in tis_to_set_to_scheduled:<EOL><INDENT>task_instance.state = State.SCHEDULED<EOL><DEDENT>task_instance_str = "<STR_LIT>".join(<EOL>[repr(x) for x in tis_to_set_to_scheduled])<EOL>session.commit()<EOL>self.log.info("<STR_LIT>", task_instance_str)<EOL><DEDENT>

If there are tasks left over in the executor,
we set them back to SCHEDULED to avoid creating hanging tasks.

:param session: session for ORM operations


def _process_dags(self, dagbag, dags, tis_out):

for dag in dags:<EOL><INDENT>dag = dagbag.get_dag(dag.dag_id)<EOL>if not dag:<EOL><INDENT>self.log.error("<STR_LIT>", dag.dag_id)<EOL>continue<EOL><DEDENT>if dag.is_paused:<EOL><INDENT>self.log.info("<STR_LIT>", dag.dag_id)<EOL>continue<EOL><DEDENT>self.log.info("<STR_LIT>", dag.dag_id)<EOL>dag_run = self.create_dag_run(dag)<EOL>if dag_run:<EOL><INDENT>expected_start_date = dag.following_schedule(dag_run.execution_date)<EOL>if expected_start_date:<EOL><INDENT>schedule_delay = dag_run.start_date - expected_start_date<EOL>Stats.timing(<EOL>'<STR_LIT>'.format(dag_id=dag.dag_id),<EOL>schedule_delay)<EOL><DEDENT>self.log.info("<STR_LIT>", dag_run)<EOL><DEDENT>self._process_task_instances(dag, tis_out)<EOL>self.manage_slas(dag)<EOL><DEDENT>

Iterates over the dags and processes them. Processing includes:

1. Create appropriate DagRun(s) in the DB.
2. Create appropriate TaskInstance(s) in the DB.
3. Send emails for tasks that have missed SLAs.

:param dagbag: a collection of DAGs to process
:type dagbag: airflow.models.DagBag
:param dags: the DAGs from the DagBag to process
:type dags: airflow.models.DAG
:param tis_out: A queue to add generated TaskInstance objects
:type tis_out: multiprocessing.Queue[TaskInstance]
:rtype: None


@provide_session<EOL><INDENT>def _process_executor_events(self, simple_dag_bag, session=None):<DEDENT>

<EOL>TI = models.TaskInstance<EOL>for key, state in list(self.executor.get_event_buffer(simple_dag_bag.dag_ids)<EOL>.items()):<EOL><INDENT>dag_id, task_id, execution_date, try_number = key<EOL>self.log.info(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>dag_id, task_id, execution_date, state, try_number<EOL>)<EOL>if state == State.FAILED or state == State.SUCCESS:<EOL><INDENT>qry = session.query(TI).filter(TI.dag_id == dag_id,<EOL>TI.task_id == task_id,<EOL>TI.execution_date == execution_date)<EOL>ti = qry.first()<EOL>if not ti:<EOL><INDENT>self.log.warning("<STR_LIT>", ti)<EOL>continue<EOL><DEDENT>if ti.try_number == try_number and ti.state == State.QUEUED:<EOL><INDENT>msg = ("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(ti, state, ti.state))<EOL>self.log.error(msg)<EOL>try:<EOL><INDENT>simple_dag = simple_dag_bag.get_dag(dag_id)<EOL>dagbag = models.DagBag(simple_dag.full_filepath)<EOL>dag = dagbag.get_dag(dag_id)<EOL>ti.task = dag.get_task(task_id)<EOL>ti.handle_failure(msg)<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.error("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>", ti)<EOL>ti.state = State.FAILED<EOL>session.merge(ti)<EOL>session.commit()<EOL><DEDENT><DEDENT><DEDENT><DEDENT>


self.executor.start()<EOL>self.log.info("<STR_LIT>")<EOL>self.reset_state_for_orphaned_tasks()<EOL>self.processor_agent.start()<EOL>execute_start_time = timezone.utcnow()<EOL>last_self_heartbeat_time = timezone.utcnow()<EOL>while True:<EOL><INDENT>self.log.debug("<STR_LIT>")<EOL>loop_start_time = time.time()<EOL>if self.using_sqlite:<EOL><INDENT>self.processor_agent.heartbeat()<EOL>self.log.debug(<EOL>"<STR_LIT>")<EOL>self.processor_agent.wait_until_finished()<EOL><DEDENT>self.log.debug("<STR_LIT>")<EOL>simple_dags = self.processor_agent.harvest_simple_dags()<EOL>self.log.debug("<STR_LIT>".format(len(simple_dags)))<EOL>simple_dag_bag = SimpleDagBag(simple_dags)<EOL>if len(simple_dags) > <NUM_LIT:0>:<EOL><INDENT>try:<EOL><INDENT>simple_dag_bag = SimpleDagBag(simple_dags)<EOL>self._change_state_for_tis_without_dagrun(simple_dag_bag,<EOL>[State.UP_FOR_RETRY],<EOL>State.FAILED)<EOL>self._change_state_for_tis_without_dagrun(simple_dag_bag,<EOL>[State.QUEUED,<EOL>State.SCHEDULED,<EOL>State.UP_FOR_RESCHEDULE],<EOL>State.NONE)<EOL>self._execute_task_instances(simple_dag_bag,<EOL>(State.SCHEDULED,))<EOL><DEDENT>except Exception as e:<EOL><INDENT>self.log.error("<STR_LIT>")<EOL>self.log.exception(e)<EOL>continue<EOL><DEDENT><DEDENT>self.log.debug("<STR_LIT>")<EOL>self.executor.heartbeat()<EOL>self._change_state_for_tasks_failed_to_execute()<EOL>self._process_executor_events(simple_dag_bag)<EOL>time_since_last_heartbeat = (timezone.utcnow() -<EOL>last_self_heartbeat_time).total_seconds()<EOL>if time_since_last_heartbeat > self.heartrate:<EOL><INDENT>self.log.debug("<STR_LIT>")<EOL>self.heartbeat()<EOL>last_self_heartbeat_time = timezone.utcnow()<EOL><DEDENT>is_unit_test = conf.getboolean('<STR_LIT>', '<STR_LIT>')<EOL>loop_end_time = time.time()<EOL>loop_duration = loop_end_time - loop_start_time<EOL>self.log.debug(<EOL>"<STR_LIT>",<EOL>loop_duration)<EOL>if not is_unit_test:<EOL><INDENT>self.log.debug("<STR_LIT>", self._processor_poll_interval)<EOL>time.sleep(self._processor_poll_interval)<EOL><DEDENT>if self.processor_agent.done:<EOL><INDENT>self._last_loop = True<EOL><DEDENT>if self._last_loop:<EOL><INDENT>self.log.info("<STR_LIT>"<EOL>"<STR_LIT>".format(self.num_runs))<EOL>break<EOL><DEDENT>if loop_duration < <NUM_LIT:1> and not is_unit_test:<EOL><INDENT>sleep_length = <NUM_LIT:1> - loop_duration<EOL>self.log.debug(<EOL>"<STR_LIT>"<EOL>.format(sleep_length))<EOL>sleep(sleep_length)<EOL><DEDENT><DEDENT>self.processor_agent.terminate()<EOL>if self.processor_agent.all_files_processed:<EOL><INDENT>self.log.info(<EOL>"<STR_LIT>",<EOL>execute_start_time.isoformat()<EOL>)<EOL>models.DAG.deactivate_stale_dags(execute_start_time)<EOL><DEDENT>self.executor.end()<EOL>settings.Session.remove()<EOL>

The actual scheduler loop. The main steps in the loop are:
    #. Harvest DAG parsing results through DagFileProcessorAgent
    #. Find and queue executable tasks
        #. Change task instance state in DB
        #. Queue tasks in executor
    #. Heartbeat executor
        #. Execute queued tasks in executor asynchronously
        #. Sync on the states of running tasks

Following is a graphic representation of these steps.

.. image:: ../docs/img/scheduler_loop.jpg

:rtype: None


@provide_session<EOL><INDENT>def process_file(self, file_path, zombies, pickle_dags=False, session=None):<DEDENT>

self.log.info("<STR_LIT>", file_path)<EOL>simple_dags = []<EOL>try:<EOL><INDENT>dagbag = models.DagBag(file_path, include_examples=False)<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.exception("<STR_LIT>", file_path)<EOL>Stats.incr('<STR_LIT>', <NUM_LIT:1>, <NUM_LIT:1>)<EOL>return []<EOL><DEDENT>if len(dagbag.dags) > <NUM_LIT:0>:<EOL><INDENT>self.log.info("<STR_LIT>", dagbag.dags.keys(), file_path)<EOL><DEDENT>else:<EOL><INDENT>self.log.warning("<STR_LIT>", file_path)<EOL>self.update_import_errors(session, dagbag)<EOL>return []<EOL><DEDENT>for dag in dagbag.dags.values():<EOL><INDENT>dag.sync_to_db()<EOL><DEDENT>paused_dag_ids = [dag.dag_id for dag in dagbag.dags.values()<EOL>if dag.is_paused]<EOL>for dag_id in dagbag.dags:<EOL><INDENT>if dag_id not in paused_dag_ids:<EOL><INDENT>dag = dagbag.get_dag(dag_id)<EOL>pickle_id = None<EOL>if pickle_dags:<EOL><INDENT>pickle_id = dag.pickle(session).id<EOL><DEDENT>simple_dags.append(SimpleDag(dag, pickle_id=pickle_id))<EOL><DEDENT><DEDENT>if len(self.dag_ids) > <NUM_LIT:0>:<EOL><INDENT>dags = [dag for dag in dagbag.dags.values()<EOL>if dag.dag_id in self.dag_ids and<EOL>dag.dag_id not in paused_dag_ids]<EOL><DEDENT>else:<EOL><INDENT>dags = [dag for dag in dagbag.dags.values()<EOL>if not dag.parent_dag and<EOL>dag.dag_id not in paused_dag_ids]<EOL><DEDENT>ti_keys_to_schedule = []<EOL>self._process_dags(dagbag, dags, ti_keys_to_schedule)<EOL>for ti_key in ti_keys_to_schedule:<EOL><INDENT>dag = dagbag.dags[ti_key[<NUM_LIT:0>]]<EOL>task = dag.get_task(ti_key[<NUM_LIT:1>])<EOL>ti = models.TaskInstance(task, ti_key[<NUM_LIT:2>])<EOL>ti.refresh_from_db(session=session, lock_for_update=True)<EOL>dep_context = DepContext(deps=QUEUE_DEPS, ignore_task_deps=True)<EOL>if ti.are_dependencies_met(<EOL>dep_context=dep_context,<EOL>session=session,<EOL>verbose=True):<EOL><INDENT>ti.state = State.SCHEDULED<EOL><DEDENT>self.log.info("<STR_LIT>", ti)<EOL>session.merge(ti)<EOL><DEDENT>session.commit()<EOL>try:<EOL><INDENT>self.update_import_errors(session, dagbag)<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.exception("<STR_LIT>")<EOL><DEDENT>try:<EOL><INDENT>dagbag.kill_zombies(zombies)<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.exception("<STR_LIT>")<EOL><DEDENT>return simple_dags<EOL>

Process a Python file containing Airflow DAGs.

This includes:

1. Execute the file and look for DAG objects in the namespace.
2. Pickle the DAG and save it to the DB (if necessary).
3. For each DAG, see what tasks should run and create appropriate task
instances in the DB.
4. Record any errors importing the file into ORM
5. Kill (in ORM) any task instances belonging to the DAGs that haven't
issued a heartbeat in a while.

Returns a list of SimpleDag objects that represent the DAGs found in
the file

:param file_path: the path to the Python file that should be executed
:type file_path: unicode
:param zombies: zombie task instances to kill.
:type zombies: list[airflow.utils.dag_processing.SimpleTaskInstance]
:param pickle_dags: whether serialize the DAGs found in the file and
    save them to the db
:type pickle_dags: bool
:return: a list of SimpleDags made from the Dags found in the file
:rtype: list[airflow.utils.dag_processing.SimpleDagBag]


self.dag = dag<EOL>self.dag_id = dag.dag_id<EOL>self.bf_start_date = start_date<EOL>self.bf_end_date = end_date<EOL>self.mark_success = mark_success<EOL>self.donot_pickle = donot_pickle<EOL>self.ignore_first_depends_on_past = ignore_first_depends_on_past<EOL>self.ignore_task_deps = ignore_task_deps<EOL>self.pool = pool<EOL>self.delay_on_limit_secs = delay_on_limit_secs<EOL>self.verbose = verbose<EOL>self.conf = conf<EOL>self.rerun_failed_tasks = rerun_failed_tasks<EOL>self.run_backwards = run_backwards<EOL>super().__init__(*args, **kwargs)<EOL>

:param dag: DAG object.
:type dag: airflow.models.DAG
:param start_date: start date for the backfill date range.
:type start_date: datetime.datetime
:param end_date: end date for the backfill date range.
:type end_date: datetime.datetime
:param mark_success: flag whether to mark the task auto success.
:type mark_success: bool
:param donot_pickle: whether pickle
:type donot_pickle: bool
:param ignore_first_depends_on_past: whether to ignore depend on past
:type ignore_first_depends_on_past: bool
:param ignore_task_deps: whether to ignore the task dependency
:type ignore_task_deps: bool
:param pool: pool to backfill
:type pool: str
:param delay_on_limit_secs:
:param verbose:
:type verbose: flag to whether display verbose message to backfill console
:param conf: a dictionary which user could pass k-v pairs for backfill
:type conf: dictionary
:param rerun_failed_tasks: flag to whether to
                           auto rerun the failed task in backfill
:type rerun_failed_tasks: bool
:param run_backwards: Whether to process the dates from most to least recent
:type run_backwards bool
:param args:
:param kwargs:


for key, ti in list(ti_status.running.items()):<EOL><INDENT>ti.refresh_from_db()<EOL>if ti.state == State.SUCCESS:<EOL><INDENT>ti_status.succeeded.add(key)<EOL>self.log.debug("<STR_LIT>", ti)<EOL>ti_status.running.pop(key)<EOL>continue<EOL><DEDENT>elif ti.state == State.SKIPPED:<EOL><INDENT>ti_status.skipped.add(key)<EOL>self.log.debug("<STR_LIT>", ti)<EOL>ti_status.running.pop(key)<EOL>continue<EOL><DEDENT>elif ti.state == State.FAILED:<EOL><INDENT>self.log.error("<STR_LIT>", ti)<EOL>ti_status.failed.add(key)<EOL>ti_status.running.pop(key)<EOL>continue<EOL><DEDENT>elif ti.state == State.UP_FOR_RETRY:<EOL><INDENT>self.log.warning("<STR_LIT>", ti)<EOL>ti_status.running.pop(key)<EOL>ti_status.to_run[key] = ti<EOL><DEDENT>elif ti.state == State.UP_FOR_RESCHEDULE:<EOL><INDENT>self.log.warning("<STR_LIT>", ti)<EOL>ti_status.running.pop(key)<EOL>ti_status.to_run[key] = ti<EOL><DEDENT>elif ti.state == State.NONE:<EOL><INDENT>self.log.warning(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>ti<EOL>)<EOL>ti.set_state(State.SCHEDULED)<EOL>ti_status.running.pop(key)<EOL>ti_status.to_run[key] = ti<EOL><DEDENT><DEDENT>

Updates the counters per state of the tasks that were running. Can re-add
to tasks to run in case required.

:param ti_status: the internal status of the backfill job tasks
:type ti_status: BackfillJob._DagRunTaskStatus


executor = self.executor<EOL>for key, state in list(executor.get_event_buffer().items()):<EOL><INDENT>if key not in running:<EOL><INDENT>self.log.warning(<EOL>"<STR_LIT>",<EOL>key, state, running.values()<EOL>)<EOL>continue<EOL><DEDENT>ti = running[key]<EOL>ti.refresh_from_db()<EOL>self.log.debug("<STR_LIT>", state, ti)<EOL>if state == State.FAILED or state == State.SUCCESS:<EOL><INDENT>if ti.state == State.RUNNING or ti.state == State.QUEUED:<EOL><INDENT>msg = ("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(ti, state, ti.state))<EOL>self.log.error(msg)<EOL>ti.handle_failure(msg)<EOL><DEDENT><DEDENT><DEDENT>

Checks if the executor agrees with the state of task instances
that are running

:param running: dict of key, task to verify


@provide_session<EOL><INDENT>def _get_dag_run(self, run_date, session=None):<DEDENT>

run_id = BackfillJob.ID_FORMAT_PREFIX.format(run_date.isoformat())<EOL>respect_dag_max_active_limit = (True<EOL>if (self.dag.schedule_interval and<EOL>not self.dag.is_subdag)<EOL>else False)<EOL>current_active_dag_count = self.dag.get_num_active_runs(external_trigger=False)<EOL>run = DagRun.find(dag_id=self.dag.dag_id,<EOL>execution_date=run_date,<EOL>session=session)<EOL>if run is not None and len(run) > <NUM_LIT:0>:<EOL><INDENT>run = run[<NUM_LIT:0>]<EOL>if run.state == State.RUNNING:<EOL><INDENT>respect_dag_max_active_limit = False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>run = None<EOL><DEDENT>if (respect_dag_max_active_limit and<EOL>current_active_dag_count >= self.dag.max_active_runs):<EOL><INDENT>return None<EOL><DEDENT>run = run or self.dag.create_dagrun(<EOL>run_id=run_id,<EOL>execution_date=run_date,<EOL>start_date=timezone.utcnow(),<EOL>state=State.RUNNING,<EOL>external_trigger=False,<EOL>session=session,<EOL>conf=self.conf,<EOL>)<EOL>run.dag = self.dag<EOL>run.state = State.RUNNING<EOL>run.run_id = run_id<EOL>run.verify_integrity(session=session)<EOL>return run<EOL>

Returns a dag run for the given run date, which will be matched to an existing
dag run if available or create a new dag run otherwise. If the max_active_runs
limit is reached, this function will return None.

:param run_date: the execution date for the dag run
:type run_date: datetime.datetime
:param session: the database session object
:type session: sqlalchemy.orm.session.Session
:return: a DagRun in state RUNNING or None


@provide_session<EOL><INDENT>def _task_instances_for_dag_run(self, dag_run, session=None):<DEDENT>

tasks_to_run = {}<EOL>if dag_run is None:<EOL><INDENT>return tasks_to_run<EOL><DEDENT>self.reset_state_for_orphaned_tasks(filter_by_dag_run=dag_run, session=session)<EOL>dag_run.refresh_from_db()<EOL>make_transient(dag_run)<EOL>for ti in dag_run.get_task_instances():<EOL><INDENT>if ti.state == State.NONE:<EOL><INDENT>ti.set_state(State.SCHEDULED, session=session)<EOL><DEDENT>if ti.state != State.REMOVED:<EOL><INDENT>tasks_to_run[ti.key] = ti<EOL><DEDENT><DEDENT>return tasks_to_run<EOL>

Returns a map of task instance key to task instance object for the tasks to
run in the given dag run.

:param dag_run: the dag run to get the tasks from
:type dag_run: airflow.models.DagRun
:param session: the database session object
:type session: sqlalchemy.orm.session.Session


executed_run_dates = []<EOL>while ((len(ti_status.to_run) > <NUM_LIT:0> or len(ti_status.running) > <NUM_LIT:0>) and<EOL>len(ti_status.deadlocked) == <NUM_LIT:0>):<EOL><INDENT>self.log.debug("<STR_LIT>")<EOL>ti_status.not_ready.clear()<EOL>@provide_session<EOL>def _per_task_process(task, key, ti, session=None):<EOL><INDENT>ti.refresh_from_db()<EOL>task = self.dag.get_task(ti.task_id)<EOL>ti.task = task<EOL>ignore_depends_on_past = (<EOL>self.ignore_first_depends_on_past and<EOL>ti.execution_date == (start_date or ti.start_date))<EOL>self.log.debug(<EOL>"<STR_LIT>", ti, ti.state)<EOL>if ti.state == State.SUCCESS:<EOL><INDENT>ti_status.succeeded.add(key)<EOL>self.log.debug("<STR_LIT>", ti)<EOL>ti_status.to_run.pop(key)<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>return<EOL><DEDENT>elif ti.state == State.SKIPPED:<EOL><INDENT>ti_status.skipped.add(key)<EOL>self.log.debug("<STR_LIT>", ti)<EOL>ti_status.to_run.pop(key)<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>return<EOL><DEDENT>elif ti.state == State.NONE:<EOL><INDENT>self.log.warning(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL>ti.set_state(State.SCHEDULED, session=session)<EOL><DEDENT>if self.rerun_failed_tasks:<EOL><INDENT>if ti.state in (State.FAILED, State.UPSTREAM_FAILED):<EOL><INDENT>self.log.error("<STR_LIT>"<EOL>"<STR_LIT>".format(ti=ti,<EOL>state=ti.state))<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>ti.set_state(State.SCHEDULED, session=session)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if ti.state in (State.FAILED, State.UPSTREAM_FAILED):<EOL><INDENT>self.log.error("<STR_LIT>"<EOL>"<STR_LIT>".format(ti=ti,<EOL>state=ti.state))<EOL>ti_status.failed.add(key)<EOL>ti_status.to_run.pop(key)<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>return<EOL><DEDENT><DEDENT>backfill_context = DepContext(<EOL>deps=RUN_DEPS,<EOL>ignore_depends_on_past=ignore_depends_on_past,<EOL>ignore_task_deps=self.ignore_task_deps,<EOL>flag_upstream_failed=True)<EOL>if ti.are_dependencies_met(<EOL>dep_context=backfill_context,<EOL>session=session,<EOL>verbose=self.verbose):<EOL><INDENT>ti.refresh_from_db(lock_for_update=True, session=session)<EOL>if ti.state in (State.SCHEDULED, State.UP_FOR_RETRY, State.UP_FOR_RESCHEDULE):<EOL><INDENT>if executor.has_task(ti):<EOL><INDENT>self.log.debug(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>ti<EOL>)<EOL><DEDENT>else:<EOL><INDENT>self.log.debug('<STR_LIT>', ti)<EOL>ti.state = State.QUEUED<EOL>ti.queued_dttm = timezone.utcnow() if not ti.queued_dttm else ti.queued_dttm<EOL>session.merge(ti)<EOL>cfg_path = None<EOL>if executor.__class__ in (executors.LocalExecutor,<EOL>executors.SequentialExecutor):<EOL><INDENT>cfg_path = tmp_configuration_copy()<EOL><DEDENT>executor.queue_task_instance(<EOL>ti,<EOL>mark_success=self.mark_success,<EOL>pickle_id=pickle_id,<EOL>ignore_task_deps=self.ignore_task_deps,<EOL>ignore_depends_on_past=ignore_depends_on_past,<EOL>pool=self.pool,<EOL>cfg_path=cfg_path)<EOL>ti_status.running[key] = ti<EOL>ti_status.to_run.pop(key)<EOL><DEDENT><DEDENT>session.commit()<EOL>return<EOL><DEDENT>if ti.state == State.UPSTREAM_FAILED:<EOL><INDENT>self.log.error("<STR_LIT>", ti)<EOL>ti_status.failed.add(key)<EOL>ti_status.to_run.pop(key)<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>return<EOL><DEDENT>if ti.state == State.UP_FOR_RETRY:<EOL><INDENT>self.log.debug(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>", ti)<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>ti_status.to_run[key] = ti<EOL>return<EOL><DEDENT>if ti.state == State.UP_FOR_RESCHEDULE:<EOL><INDENT>self.log.debug(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>", ti)<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>ti_status.to_run[key] = ti<EOL>return<EOL><DEDENT>self.log.debug('<STR_LIT>', ti)<EOL>ti_status.not_ready.add(key)<EOL><DEDENT>non_pool_slots = conf.getint('<STR_LIT>', '<STR_LIT>')<EOL>try:<EOL><INDENT>for task in self.dag.topological_sort():<EOL><INDENT>for key, ti in list(ti_status.to_run.items()):<EOL><INDENT>if task.task_id != ti.task_id:<EOL><INDENT>continue<EOL><DEDENT>if task.pool:<EOL><INDENT>pool = session.query(models.Pool).filter(models.Pool.pool == task.pool).first()<EOL>if not pool:<EOL><INDENT>raise PoolNotFound('<STR_LIT>'.format(task.pool))<EOL><DEDENT>open_slots = pool.open_slots(session=session)<EOL>if open_slots <= <NUM_LIT:0>:<EOL><INDENT>raise NoAvailablePoolSlot(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(<EOL>open_slots, task.pool))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if non_pool_slots <= <NUM_LIT:0>:<EOL><INDENT>raise NoAvailablePoolSlot(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>non_pool_slots -= <NUM_LIT:1><EOL><DEDENT>num_running_tasks = DAG.get_num_task_instances(<EOL>self.dag_id,<EOL>states=(State.QUEUED, State.RUNNING))<EOL>if num_running_tasks >= self.dag.concurrency:<EOL><INDENT>raise DagConcurrencyLimitReached(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>_per_task_process(task, key, ti)<EOL><DEDENT><DEDENT><DEDENT>except (NoAvailablePoolSlot, DagConcurrencyLimitReached) as e:<EOL><INDENT>self.log.debug(e)<EOL><DEDENT>self.heartbeat()<EOL>executor.heartbeat()<EOL>if (ti_status.not_ready and<EOL>ti_status.not_ready == set(ti_status.to_run) and<EOL>len(ti_status.running) == <NUM_LIT:0>):<EOL><INDENT>self.log.warning(<EOL>"<STR_LIT>",<EOL>ti_status.to_run.values()<EOL>)<EOL>ti_status.deadlocked.update(ti_status.to_run.values())<EOL>ti_status.to_run.clear()<EOL><DEDENT>self._manage_executor_state(ti_status.running)<EOL>self._update_counters(ti_status=ti_status)<EOL>_dag_runs = ti_status.active_runs[:]<EOL>for run in _dag_runs:<EOL><INDENT>run.update_state(session=session)<EOL>if run.state in State.finished():<EOL><INDENT>ti_status.finished_runs += <NUM_LIT:1><EOL>ti_status.active_runs.remove(run)<EOL>executed_run_dates.append(run.execution_date)<EOL><DEDENT><DEDENT>self._log_progress(ti_status)<EOL><DEDENT>return executed_run_dates<EOL>

Process a set of task instances from a set of dag runs. Special handling is done
to account for different task instance states that could be present when running
them in a backfill process.

:param ti_status: the internal status of the job
:type ti_status: BackfillJob._DagRunTaskStatus
:param executor: the executor to run the task instances
:type executor: BaseExecutor
:param pickle_id: the pickle_id if dag is pickled, None otherwise
:type pickle_id: int
:param start_date: the start date of the backfill job
:type start_date: datetime.datetime
:param session: the current session object
:type session: sqlalchemy.orm.session.Session
:return: the list of execution_dates for the finished dag runs
:rtype: list


@provide_session<EOL><INDENT>def _execute_for_run_dates(self, run_dates, ti_status, executor, pickle_id,<EOL>start_date, session=None):<DEDENT>

for next_run_date in run_dates:<EOL><INDENT>dag_run = self._get_dag_run(next_run_date, session=session)<EOL>tis_map = self._task_instances_for_dag_run(dag_run,<EOL>session=session)<EOL>if dag_run is None:<EOL><INDENT>continue<EOL><DEDENT>ti_status.active_runs.append(dag_run)<EOL>ti_status.to_run.update(tis_map or {})<EOL><DEDENT>processed_dag_run_dates = self._process_backfill_task_instances(<EOL>ti_status=ti_status,<EOL>executor=executor,<EOL>pickle_id=pickle_id,<EOL>start_date=start_date,<EOL>session=session)<EOL>ti_status.executed_dag_run_dates.update(processed_dag_run_dates)<EOL>

Computes the dag runs and their respective task instances for
the given run dates and executes the task instances.
Returns a list of execution dates of the dag runs that were executed.

:param run_dates: Execution dates for dag runs
:type run_dates: list
:param ti_status: internal BackfillJob status structure to tis track progress
:type ti_status: BackfillJob._DagRunTaskStatus
:param executor: the executor to use, it must be previously started
:type executor: BaseExecutor
:param pickle_id: numeric id of the pickled dag, None if not pickled
:type pickle_id: int
:param start_date: backfill start date
:type start_date: datetime.datetime
:param session: the current session object
:type session: sqlalchemy.orm.session.Session


@provide_session<EOL><INDENT>def _set_unfinished_dag_runs_to_failed(self, dag_runs, session=None):<DEDENT>

for dag_run in dag_runs:<EOL><INDENT>dag_run.update_state()<EOL>if dag_run.state not in State.finished():<EOL><INDENT>dag_run.set_state(State.FAILED)<EOL><DEDENT>session.merge(dag_run)<EOL><DEDENT>

Go through the dag_runs and update the state based on the task_instance state.
Then set DAG runs that are not finished to failed.

:param dag_runs: DAG runs
:param session: session
:return: None


ti_status = BackfillJob._DagRunTaskStatus()<EOL>start_date = self.bf_start_date<EOL>run_dates = self.dag.get_run_dates(start_date=start_date,<EOL>end_date=self.bf_end_date)<EOL>if self.run_backwards:<EOL><INDENT>tasks_that_depend_on_past = [t.task_id for t in self.dag.task_dict.values() if t.depends_on_past]<EOL>if tasks_that_depend_on_past:<EOL><INDENT>raise AirflowException(<EOL>'<STR_LIT>'.format(<EOL>"<STR_LIT:U+002C>".join(tasks_that_depend_on_past)))<EOL><DEDENT>run_dates = run_dates[::-<NUM_LIT:1>]<EOL><DEDENT>if len(run_dates) == <NUM_LIT:0>:<EOL><INDENT>self.log.info("<STR_LIT>")<EOL>return<EOL><DEDENT>pickle_id = None<EOL>if not self.donot_pickle and self.executor.__class__ not in (<EOL>executors.LocalExecutor, executors.SequentialExecutor):<EOL><INDENT>pickle = DagPickle(self.dag)<EOL>session.add(pickle)<EOL>session.commit()<EOL>pickle_id = pickle.id<EOL><DEDENT>executor = self.executor<EOL>executor.start()<EOL>ti_status.total_runs = len(run_dates)  <EOL>try:<EOL><INDENT>remaining_dates = ti_status.total_runs<EOL>while remaining_dates > <NUM_LIT:0>:<EOL><INDENT>dates_to_process = [run_date for run_date in run_dates<EOL>if run_date not in ti_status.executed_dag_run_dates]<EOL>self._execute_for_run_dates(run_dates=dates_to_process,<EOL>ti_status=ti_status,<EOL>executor=executor,<EOL>pickle_id=pickle_id,<EOL>start_date=start_date,<EOL>session=session)<EOL>remaining_dates = (<EOL>ti_status.total_runs - len(ti_status.executed_dag_run_dates)<EOL>)<EOL>err = self._collect_errors(ti_status=ti_status, session=session)<EOL>if err:<EOL><INDENT>raise AirflowException(err)<EOL><DEDENT>if remaining_dates > <NUM_LIT:0>:<EOL><INDENT>self.log.info(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>self.dag_id<EOL>)<EOL>time.sleep(self.delay_on_limit_secs)<EOL><DEDENT><DEDENT><DEDENT>except (KeyboardInterrupt, SystemExit):<EOL><INDENT>self.log.warning("<STR_LIT>")<EOL>self._set_unfinished_dag_runs_to_failed(ti_status.active_runs)<EOL><DEDENT>finally:<EOL><INDENT>session.commit()<EOL>executor.end()<EOL><DEDENT>self.log.info("<STR_LIT>")<EOL>

Initializes all components required to run a dag for a specified date range and
calls helper method to execute the tasks.


if self.terminating:<EOL><INDENT>self.task_runner.terminate()<EOL>return<EOL><DEDENT>self.task_instance.refresh_from_db()<EOL>ti = self.task_instance<EOL>fqdn = get_hostname()<EOL>same_hostname = fqdn == ti.hostname<EOL>same_process = ti.pid == os.getpid()<EOL>if ti.state == State.RUNNING:<EOL><INDENT>if not same_hostname:<EOL><INDENT>self.log.warning("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT:%s>", ti.hostname, fqdn)<EOL>raise AirflowException("<STR_LIT>")<EOL><DEDENT>elif not same_process:<EOL><INDENT>current_pid = os.getpid()<EOL>self.log.warning("<STR_LIT>"<EOL>"<STR_LIT>", ti.pid, current_pid)<EOL>raise AirflowException("<STR_LIT>")<EOL><DEDENT><DEDENT>elif (<EOL>self.task_runner.return_code() is None and<EOL>hasattr(self.task_runner, '<STR_LIT>')<EOL>):<EOL><INDENT>self.log.warning(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>ti.state<EOL>)<EOL>self.task_runner.terminate()<EOL>self.terminating = True<EOL><DEDENT>

Self destruct task if state has been moved away from running externally


backend = _get_backend()<EOL>@wraps(func)<EOL>def wrapper(self, context, *args, **kwargs):<EOL><INDENT>self.log.debug("<STR_LIT>",<EOL>backend, self.inlets, self.outlets)<EOL>ret_val = func(self, context, *args, **kwargs)<EOL>outlets = [x.as_dict() for x in self.outlets]<EOL>inlets = [x.as_dict() for x in self.inlets]<EOL>if len(self.outlets) > <NUM_LIT:0>:<EOL><INDENT>self.xcom_push(context,<EOL>key=PIPELINE_OUTLETS,<EOL>value=outlets,<EOL>execution_date=context['<STR_LIT>'].execution_date)<EOL><DEDENT>if len(self.inlets) > <NUM_LIT:0>:<EOL><INDENT>self.xcom_push(context,<EOL>key=PIPELINE_INLETS,<EOL>value=inlets,<EOL>execution_date=context['<STR_LIT>'].execution_date)<EOL><DEDENT>if backend:<EOL><INDENT>backend.send_lineage(operator=self, inlets=self.inlets,<EOL>outlets=self.outlets, context=context)<EOL><DEDENT>return ret_val<EOL><DEDENT>return wrapper<EOL>

Saves the lineage to XCom and if configured to do so sends it
to the backend.


@wraps(func)<EOL>def wrapper(self, context, *args, **kwargs):<EOL><INDENT>self.log.debug("<STR_LIT>")<EOL>task_ids = set(self._inlets['<STR_LIT>']).intersection(<EOL>self.get_flat_relative_ids(upstream=True)<EOL>)<EOL>if task_ids:<EOL><INDENT>inlets = self.xcom_pull(context,<EOL>task_ids=task_ids,<EOL>dag_id=self.dag_id,<EOL>key=PIPELINE_OUTLETS)<EOL>inlets = [item for sublist in inlets if sublist for item in sublist]<EOL>inlets = [DataSet.map_type(i['<STR_LIT>'])(data=i['<STR_LIT>'])<EOL>for i in inlets]<EOL>self.inlets.extend(inlets)<EOL><DEDENT>if self._inlets['<STR_LIT>']:<EOL><INDENT>task_ids = set(self._inlets['<STR_LIT>']).symmetric_difference(<EOL>self.upstream_task_ids<EOL>)<EOL>inlets = self.xcom_pull(context,<EOL>task_ids=task_ids,<EOL>dag_id=self.dag_id,<EOL>key=PIPELINE_OUTLETS)<EOL>inlets = [item for sublist in inlets if sublist for item in sublist]<EOL>inlets = [DataSet.map_type(i['<STR_LIT>'])(data=i['<STR_LIT>'])<EOL>for i in inlets]<EOL>self.inlets.extend(inlets)<EOL><DEDENT>if len(self._inlets['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>self.inlets.extend(self._inlets['<STR_LIT>'])<EOL><DEDENT>if len(self._outlets['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>self.outlets.extend(self._outlets['<STR_LIT>'])<EOL><DEDENT>self.log.debug("<STR_LIT>", self.inlets, self.outlets)<EOL>for dataset in chain(self.inlets, self.outlets):<EOL><INDENT>dataset.set_context(context)<EOL><DEDENT>return func(self, context, *args, **kwargs)<EOL><DEDENT>return wrapper<EOL>

Prepares the lineage inlets and outlets. Inlets can be:

* "auto" -> picks up any outlets from direct upstream tasks that have outlets defined, as such that
  if A -> B -> C and B does not have outlets but A does, these are provided as inlets.
* "list of task_ids" -> picks up outlets from the upstream task_ids
* "list of datasets" -> manually defined list of DataSet


def send_lineage(self,<EOL>operator=None, inlets=None, outlets=None, context=None):

raise NotImplementedError()<EOL>

Sends lineage metadata to a backend
:param operator: the operator executing a transformation on the inlets and outlets
:param inlets: the inlets to this operator
:param outlets: the outlets from this operator
:param context: the current context of the task instance


return value.utcoffset() is not None<EOL>

Determine if a given datetime.datetime is aware.
The concept is defined in Python's docs:
http://docs.python.org/library/datetime.html#datetime.tzinfo
Assuming value.tzinfo is either None or a proper datetime.tzinfo,
value.utcoffset() implements the appropriate logic.


return value.utcoffset() is None<EOL>

Determine if a given datetime.datetime is naive.
The concept is defined in Python's docs:
http://docs.python.org/library/datetime.html#datetime.tzinfo
Assuming value.tzinfo is either None or a proper datetime.tzinfo,
value.utcoffset() implements the appropriate logic.


<EOL>d = dt.datetime.utcnow()<EOL>d = d.replace(tzinfo=utc)<EOL>return d<EOL>

Get the current date and time in UTC
:return:


<EOL>d = dt.datetime(<NUM_LIT>, <NUM_LIT:1>, <NUM_LIT:1>)<EOL>d = d.replace(tzinfo=utc)<EOL>return d<EOL>

Gets the epoch in the users timezone
:return:


if not value:<EOL><INDENT>return value<EOL><DEDENT>if not is_localized(value):<EOL><INDENT>value = pendulum.instance(value, TIMEZONE)<EOL><DEDENT>return value.astimezone(utc)<EOL>

Returns the datetime with the default timezone added if timezone
information was not associated
:param value: datetime
:return: datetime with tzinfo


if timezone is None:<EOL><INDENT>timezone = TIMEZONE<EOL><DEDENT>if is_localized(value):<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>" % value)<EOL><DEDENT>if hasattr(value, '<STR_LIT>'):<EOL><INDENT>value = value.replace(fold=<NUM_LIT:1>)<EOL><DEDENT>if hasattr(timezone, '<STR_LIT>'):<EOL><INDENT>return timezone.localize(value)<EOL><DEDENT>elif hasattr(timezone, '<STR_LIT>'):<EOL><INDENT>return timezone.convert(value)<EOL><DEDENT>else:<EOL><INDENT>return value.replace(tzinfo=timezone)<EOL><DEDENT>

Make a naive datetime.datetime in a given time zone aware.

:param value: datetime
:param timezone: timezone
:return: localized datetime in settings.TIMEZONE or timezone


if timezone is None:<EOL><INDENT>timezone = TIMEZONE<EOL><DEDENT>if is_naive(value):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>o = value.astimezone(timezone)<EOL>naive = dt.datetime(o.year,<EOL>o.month,<EOL>o.day,<EOL>o.hour,<EOL>o.minute,<EOL>o.second,<EOL>o.microsecond)<EOL>return naive<EOL>

Make an aware datetime.datetime naive in a given time zone.

:param value: datetime
:param timezone: timezone
:return: naive datetime


if '<STR_LIT>' not in kwargs:<EOL><INDENT>kwargs['<STR_LIT>'] = TIMEZONE<EOL><DEDENT>return dt.datetime(*args, **kwargs)<EOL>

Wrapper around datetime.datetime that adds settings.TIMEZONE if tzinfo not specified

:return: datetime.datetime


return pendulum.parse(string, tz=timezone or TIMEZONE)<EOL>

Parse a time string and return an aware datetime
:param string: time string


params = dict()<EOL>if in_env_var_format:<EOL><INDENT>name_format = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>name_format = '<STR_LIT:default>'<EOL><DEDENT>task_instance = context.get('<STR_LIT>')<EOL>if task_instance and task_instance.dag_id:<EOL><INDENT>params[AIRFLOW_VAR_NAME_FORMAT_MAPPING['<STR_LIT>'][<EOL>name_format]] = task_instance.dag_id<EOL><DEDENT>if task_instance and task_instance.task_id:<EOL><INDENT>params[AIRFLOW_VAR_NAME_FORMAT_MAPPING['<STR_LIT>'][<EOL>name_format]] = task_instance.task_id<EOL><DEDENT>if task_instance and task_instance.execution_date:<EOL><INDENT>params[<EOL>AIRFLOW_VAR_NAME_FORMAT_MAPPING['<STR_LIT>'][<EOL>name_format]] = task_instance.execution_date.isoformat()<EOL><DEDENT>dag_run = context.get('<STR_LIT>')<EOL>if dag_run and dag_run.run_id:<EOL><INDENT>params[AIRFLOW_VAR_NAME_FORMAT_MAPPING['<STR_LIT>'][<EOL>name_format]] = dag_run.run_id<EOL><DEDENT>return params<EOL>

Given a context, this function provides a dictionary of values that can be used to
externally reconstruct relations between dags, dag_runs, tasks and task_instances.
Default to abc.def.ghi format and can be made to ABC_DEF_GHI format if
in_env_var_format is set to True.

:param context: The context for the task_instance of interest.
:type context: dict
:param in_env_var_format: If returned vars should be in ABC_DEF_GHI format.
:type in_env_var_format: bool
:return: task_instance context as dict.


<EOL>if self.closed:<EOL><INDENT>return<EOL><DEDENT>super().close()<EOL>if not self.upload_on_close:<EOL><INDENT>return<EOL><DEDENT>local_loc = os.path.join(self.local_base, self.log_relative_path)<EOL>remote_loc = os.path.join(self.remote_base, self.log_relative_path)<EOL>if os.path.exists(local_loc):<EOL><INDENT>with open(local_loc, '<STR_LIT:r>') as logfile:<EOL><INDENT>log = logfile.read()<EOL><DEDENT>self.s3_write(log, remote_loc)<EOL><DEDENT>self.closed = True<EOL>

Close and upload local log file to remote storage S3.


def _read(self, ti, try_number, metadata=None):

<EOL>log_relative_path = self._render_filename(ti, try_number)<EOL>remote_loc = os.path.join(self.remote_base, log_relative_path)<EOL>if self.s3_log_exists(remote_loc):<EOL><INDENT>remote_log = self.s3_read(remote_loc, return_error=True)<EOL>log = '<STR_LIT>'.format(<EOL>remote_loc, remote_log)<EOL>return log, {'<STR_LIT>': True}<EOL><DEDENT>else:<EOL><INDENT>return super()._read(ti, try_number)<EOL><DEDENT>

Read logs of given task instance and try_number from S3 remote storage.
If failed, read the log from task instance host machine.
:param ti: task instance object
:param try_number: task instance try_number to read logs from
:param metadata: log metadata,
                 can be used for steaming log reading and auto-tailing.


try:<EOL><INDENT>return self.hook.get_key(remote_log_location) is not None<EOL><DEDENT>except Exception:<EOL><INDENT>pass<EOL><DEDENT>return False<EOL>

Check if remote_log_location exists in remote storage
:param remote_log_location: log's location in remote storage
:return: True if location exists else False


def s3_read(self, remote_log_location, return_error=False):

try:<EOL><INDENT>return self.hook.read_key(remote_log_location)<EOL><DEDENT>except Exception:<EOL><INDENT>msg = '<STR_LIT>'.format(remote_log_location)<EOL>self.log.exception(msg)<EOL>if return_error:<EOL><INDENT>return msg<EOL><DEDENT><DEDENT>

Returns the log found at the remote_log_location. Returns '' if no
logs are found or there is an error.
:param remote_log_location: the log's location in remote storage
:type remote_log_location: str (path)
:param return_error: if True, returns a string error message if an
    error occurs. Otherwise returns '' when an error occurs.
:type return_error: bool


def s3_write(self, log, remote_log_location, append=True):

if append and self.s3_log_exists(remote_log_location):<EOL><INDENT>old_log = self.s3_read(remote_log_location)<EOL>log = '<STR_LIT:\n>'.join([old_log, log]) if old_log else log<EOL><DEDENT>try:<EOL><INDENT>self.hook.load_string(<EOL>log,<EOL>key=remote_log_location,<EOL>replace=True,<EOL>encrypt=configuration.conf.getboolean('<STR_LIT>', '<STR_LIT>'),<EOL>)<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.exception('<STR_LIT>', remote_log_location)<EOL><DEDENT>

Writes the log to the remote_log_location. Fails silently if no hook
was created.
:param log: the log to write to the remote_log_location
:type log: str
:param remote_log_location: the log's location in remote storage
:type remote_log_location: str (path)
:param append: if False, any existing log file is overwritten. If True,
    the new log is appended to any existing logs.
:type append: bool


<EOL>if self.closed:<EOL><INDENT>return<EOL><DEDENT>super().close()<EOL>if not self.upload_on_close:<EOL><INDENT>return<EOL><DEDENT>local_loc = os.path.join(self.local_base, self.log_relative_path)<EOL>remote_loc = os.path.join(self.remote_base, self.log_relative_path)<EOL>if os.path.exists(local_loc):<EOL><INDENT>with open(local_loc, '<STR_LIT:r>') as logfile:<EOL><INDENT>log = logfile.read()<EOL><DEDENT>self.wasb_write(log, remote_loc, append=True)<EOL>if self.delete_local_copy:<EOL><INDENT>shutil.rmtree(os.path.dirname(local_loc))<EOL><DEDENT><DEDENT>self.closed = True<EOL>

Close and upload local log file to remote storage Wasb.


<EOL>log_relative_path = self._render_filename(ti, try_number)<EOL>remote_loc = os.path.join(self.remote_base, log_relative_path)<EOL>if self.wasb_log_exists(remote_loc):<EOL><INDENT>remote_log = self.wasb_read(remote_loc, return_error=True)<EOL>log = '<STR_LIT>'.format(<EOL>remote_loc, remote_log)<EOL>return log, {'<STR_LIT>': True}<EOL><DEDENT>else:<EOL><INDENT>return super()._read(ti, try_number)<EOL><DEDENT>

Read logs of given task instance and try_number from Wasb remote storage.
If failed, read the log from task instance host machine.
:param ti: task instance object
:param try_number: task instance try_number to read logs from
:param metadata: log metadata,
                 can be used for steaming log reading and auto-tailing.


try:<EOL><INDENT>return self.hook.check_for_blob(self.wasb_container, remote_log_location)<EOL><DEDENT>except Exception:<EOL><INDENT>pass<EOL><DEDENT>return False<EOL>


def wasb_read(self, remote_log_location, return_error=False):

try:<EOL><INDENT>return self.hook.read_file(self.wasb_container, remote_log_location)<EOL><DEDENT>except AzureHttpError:<EOL><INDENT>msg = '<STR_LIT>'.format(remote_log_location)<EOL>self.log.exception(msg)<EOL>if return_error:<EOL><INDENT>return msg<EOL><DEDENT><DEDENT>


def wasb_write(self, log, remote_log_location, append=True):

if append and self.wasb_log_exists(remote_log_location):<EOL><INDENT>old_log = self.wasb_read(remote_log_location)<EOL>log = '<STR_LIT:\n>'.join([old_log, log]) if old_log else log<EOL><DEDENT>try:<EOL><INDENT>self.hook.load_string(<EOL>log,<EOL>self.wasb_container,<EOL>remote_log_location,<EOL>)<EOL><DEDENT>except AzureHttpError:<EOL><INDENT>self.log.exception('<STR_LIT>',<EOL>remote_log_location)<EOL><DEDENT>


_logger = logger<EOL>while _logger:<EOL><INDENT>for handler in _logger.handlers:<EOL><INDENT>try:<EOL><INDENT>handler.set_context(value)<EOL><DEDENT>except AttributeError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>if _logger.propagate is True:<EOL><INDENT>_logger = _logger.parent<EOL><DEDENT>else:<EOL><INDENT>_logger = None<EOL><DEDENT><DEDENT>

Walks the tree of loggers and tries to set the context for each handler
:param logger: logger
:param value: value to set


self.logger = logger<EOL>self.level = level<EOL>self._buffer = str()<EOL>

:param log: The log level method to write to, ie. log.debug, log.warning
:return:


return False<EOL>

Returns False to indicate that the stream is not closed (as it will be
open for the duration of Airflow's lifecycle).

For compatibility with the io.IOBase interface.


if not message.endswith("<STR_LIT:\n>"):<EOL><INDENT>self._buffer += message<EOL><DEDENT>else:<EOL><INDENT>self._buffer += message<EOL>self.logger.log(self.level, self._buffer.rstrip())<EOL>self._buffer = str()<EOL><DEDENT>

Do whatever it takes to actually log the specified logging record
:param message: message to log


if len(self._buffer) > <NUM_LIT:0>:<EOL><INDENT>self.logger.log(self.level, self._buffer)<EOL>self._buffer = str()<EOL><DEDENT>

Ensure all logging output has been flushed


return False<EOL>

Returns False to indicate the fd is not connected to a tty(-like) device.
For compatibility reasons.


def __init__(self, base_log_folder, filename_template):

super().__init__()<EOL>self.handler = None<EOL>self.local_base = base_log_folder<EOL>self.filename_template, self.filename_jinja_template =parse_template_string(filename_template)<EOL>

:param base_log_folder: Base log folder to place logs.
:param filename_template: template filename string


local_loc = self._init_file(ti)<EOL>self.handler = logging.FileHandler(local_loc)<EOL>self.handler.setFormatter(self.formatter)<EOL>self.handler.setLevel(self.level)<EOL>

Provide task_instance context to airflow task handler.
:param ti: task instance object


<EOL>log_relative_path = self._render_filename(ti, try_number)<EOL>location = os.path.join(self.local_base, log_relative_path)<EOL>log = "<STR_LIT>"<EOL>if os.path.exists(location):<EOL><INDENT>try:<EOL><INDENT>with open(location) as f:<EOL><INDENT>log += "<STR_LIT>".format(location)<EOL>log += "<STR_LIT>".join(f.readlines())<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>log = "<STR_LIT>".format(location)<EOL>log += "<STR_LIT>".format(str(e))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>url = os.path.join(<EOL>"<STR_LIT>", log_relative_path<EOL>).format(<EOL>ti=ti,<EOL>worker_log_server_port=conf.get('<STR_LIT>', '<STR_LIT>')<EOL>)<EOL>log += "<STR_LIT>".format(location)<EOL>log += "<STR_LIT>".format(url)<EOL>try:<EOL><INDENT>timeout = None  <EOL>try:<EOL><INDENT>timeout = conf.getint('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>except (AirflowConfigException, ValueError):<EOL><INDENT>pass<EOL><DEDENT>response = requests.get(url, timeout=timeout)<EOL>response.raise_for_status()<EOL>log += '<STR_LIT:\n>' + response.text<EOL><DEDENT>except Exception as e:<EOL><INDENT>log += "<STR_LIT>".format(str(e))<EOL><DEDENT><DEDENT>return log, {'<STR_LIT>': True}<EOL>

Template method that contains custom logic of reading
logs given the try_number.
:param ti: task instance record
:param try_number: current try_number to read log from
:param metadata: log metadata,
                 can be used for steaming log reading and auto-tailing.
:return: log message as a string and metadata.