MAX_LABEL_LEN = <NUM_LIT><EOL>safe_label = re.sub(r'<STR_LIT>', '<STR_LIT>', string)<EOL>if len(safe_label) > MAX_LABEL_LEN or string != safe_label:<EOL><INDENT>safe_hash = hashlib.md5(string.encode()).hexdigest()[:<NUM_LIT:9>]<EOL>safe_label = safe_label[:MAX_LABEL_LEN - len(safe_hash) - <NUM_LIT:1>] + "<STR_LIT:->" + safe_hash<EOL><DEDENT>return safe_label<EOL>

Valid label values must be 63 characters or less and must be empty or begin and
end with an alphanumeric character ([a-z0-9A-Z]) with dashes (-), underscores (_),
dots (.), and alphanumerics between.

If the label value is then greater than 63 chars once made safe, or differs in any
way from the original value sent to this function, then we need to truncate to
53chars, and append it with a unique hash.


super().execute(context)<EOL>return self._messages<EOL>

Overridden to allow messages to be passed


if current_num_objects > self.previous_num_objects:<EOL><INDENT>self.log.info(<EOL>'''<STR_LIT>'''.format(os.path.join(self.bucket, self.prefix)))<EOL>self.last_activity_time = get_time()<EOL>self.inactivity_seconds = <NUM_LIT:0><EOL>self.previous_num_objects = current_num_objects<EOL><DEDENT>elif current_num_objects < self.previous_num_objects:<EOL><INDENT>if self.allow_delete:<EOL><INDENT>self.previous_num_objects = current_num_objects<EOL>self.last_activity_time = get_time()<EOL>self.log.warning(<EOL>'''<STR_LIT>'''<EOL>)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError(<EOL>'''<STR_LIT>'''.format(os.path.join(self.bucket, self.prefix))<EOL>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if self.last_activity_time:<EOL><INDENT>self.inactivity_seconds = (<EOL>get_time() - self.last_activity_time).total_seconds()<EOL><DEDENT>else:<EOL><INDENT>self.last_activity_time = get_time()<EOL>self.inactivity_seconds = <NUM_LIT:0><EOL><DEDENT>if self.inactivity_seconds >= self.inactivity_period:<EOL><INDENT>if current_num_objects >= self.min_objects:<EOL><INDENT>self.log.info(<EOL>'''<STR_LIT>'''.format(<EOL>current_num_objects,<EOL>os.path.join(self.bucket, self.prefix),<EOL>self.inactivity_period))<EOL>return True<EOL><DEDENT>warn_msg ='''<STR_LIT>'''.format(<EOL>os.path.join(self.bucket, self.prefix))<EOL>self.log.warning(warn_msg)<EOL>return False<EOL><DEDENT>return False<EOL><DEDENT>

Checks whether new objects have been uploaded and the inactivity_period
has passed and updates the state of the sensor accordingly.

:param current_num_objects: number of objects in bucket during last poke.
:type current_num_objects: int


return FTPSHook(ftp_conn_id=self.ftp_conn_id)<EOL>


try:<EOL><INDENT>matches = self.error_code_pattern.match(str(e))<EOL>code = int(matches.group(<NUM_LIT:0>))<EOL>return code<EOL><DEDENT>except ValueError:<EOL><INDENT>return e<EOL><DEDENT>


self.log.info('<STR_LIT>', self.attachment_name)<EOL>with ImapHook(imap_conn_id=self.conn_id) as imap_hook:<EOL><INDENT>return imap_hook.has_mail_attachment(<EOL>name=self.attachment_name,<EOL>mail_folder=self.mail_folder,<EOL>check_regex=self.check_regex<EOL>)<EOL><DEDENT>

Pokes for a mail attachment on the mail server.

:param context: The context that is being provided when poking.
:type context: dict
:return: True if attachment with the given name is present and False if not.
:rtype: bool


sqs_hook = SQSHook(aws_conn_id=self.aws_conn_id)<EOL>sqs_conn = sqs_hook.get_conn()<EOL>self.log.info('<STR_LIT>', self.sqs_queue)<EOL>messages = sqs_conn.receive_message(QueueUrl=self.sqs_queue,<EOL>MaxNumberOfMessages=self.max_messages,<EOL>WaitTimeSeconds=self.wait_time_seconds)<EOL>self.log.info("<STR_LIT>", str(messages))<EOL>if '<STR_LIT>' in messages and len(messages['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>entries = [{'<STR_LIT>': message['<STR_LIT>'], '<STR_LIT>': message['<STR_LIT>']}<EOL>for message in messages['<STR_LIT>']]<EOL>result = sqs_conn.delete_message_batch(QueueUrl=self.sqs_queue,<EOL>Entries=entries)<EOL>if '<STR_LIT>' in result:<EOL><INDENT>context['<STR_LIT>'].xcom_push(key='<STR_LIT>', value=messages)<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>raise AirflowException(<EOL>'<STR_LIT>' + str(result) + '<STR_LIT>' + str(messages))<EOL><DEDENT><DEDENT>return False<EOL>

Check for message on subscribed queue and write to xcom the message with key ``messages``

:param context: the context object
:type context: dict
:return: ``True`` if message is available or ``False``


bash_command = self.bash_command<EOL>self.log.info("<STR_LIT>", gettempdir())<EOL>with TemporaryDirectory(prefix='<STR_LIT>') as tmp_dir:<EOL><INDENT>with NamedTemporaryFile(dir=tmp_dir, prefix=self.task_id) as f:<EOL><INDENT>f.write(bytes(bash_command, '<STR_LIT>'))<EOL>f.flush()<EOL>fname = f.name<EOL>script_location = tmp_dir + "<STR_LIT:/>" + fname<EOL>self.log.info("<STR_LIT>", script_location)<EOL>self.log.info("<STR_LIT>", bash_command)<EOL>sp = Popen(<EOL>['<STR_LIT>', fname],<EOL>stdout=PIPE, stderr=STDOUT,<EOL>close_fds=True, cwd=tmp_dir,<EOL>env=self.env, preexec_fn=os.setsid)<EOL>self.sp = sp<EOL>self.log.info("<STR_LIT>")<EOL>line = '<STR_LIT>'<EOL>for line in iter(sp.stdout.readline, b'<STR_LIT>'):<EOL><INDENT>line = line.decode(self.output_encoding).strip()<EOL>self.log.info(line)<EOL><DEDENT>sp.wait()<EOL>self.log.info("<STR_LIT>", sp.returncode)<EOL>return not sp.returncode<EOL><DEDENT><DEDENT>

Execute the bash command in a temporary directory
which will be cleaned afterwards


def _get_field(self, extras, field, default=None):

long_f = '<STR_LIT>'.format(field)<EOL>if long_f in extras:<EOL><INDENT>return extras[long_f]<EOL><DEDENT>else:<EOL><INDENT>self.log.info('<STR_LIT>', field)<EOL>return default<EOL><DEDENT>

Fetches a field from extras, and returns it. This is some Airflow
magic. The google_cloud_platform hook type adds custom UI elements
to the hook page, which allow admins to specify service_account,
key_path, etc. They get formatted as shown below.


hook = WasbHook(wasb_conn_id=self.wasb_conn_id)<EOL>self.log.info(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'.format(self.file_path, self.container_name, self.blob_name)<EOL>)<EOL>hook.load_file(self.file_path, self.container_name,<EOL>self.blob_name, **self.load_options)<EOL>


return {DAY: field_date.day, MONTH: field_date.month, YEAR: field_date.year}<EOL>

Convert native python ``datetime.date`` object  to a format supported by the API


<EOL>match = re.search(r'<STR_LIT>', job_id)<EOL>if match and match.start() == <NUM_LIT:0>:<EOL><INDENT>job = '<STR_LIT>'.format(job_id)<EOL><DEDENT>else:<EOL><INDENT>job = job_id<EOL><DEDENT>tracker = <NUM_LIT:0><EOL>cleansed_job_id = '<STR_LIT>'<EOL>for m in re.finditer(r'<STR_LIT>', job):<EOL><INDENT>cleansed_job_id += re.sub(r'<STR_LIT>', '<STR_LIT:_>',<EOL>job[tracker:m.start()])<EOL>cleansed_job_id += job[m.start():m.end()]<EOL>tracker = m.end()<EOL><DEDENT>cleansed_job_id += re.sub(r'<STR_LIT>', '<STR_LIT:_>', job[tracker:])<EOL>return cleansed_job_id<EOL>

Replaces invalid MLEngine job_id characters with '_'.

This also adds a leading 'z' in case job_id starts with an invalid
character.

Args:
    job_id: A job_id str that may have invalid characters.

Returns:
    A valid job_id representation.


self.hook = OpsgenieAlertHook(self.opsgenie_conn_id)<EOL>self.hook.execute(self._build_opsgenie_payload())<EOL>

Call the OpsgenieAlertHook to post message


def prepare_additional_parameters(additional_properties, language_hints, web_detection_params):

if language_hints is None and web_detection_params is None:<EOL><INDENT>return additional_properties<EOL><DEDENT>if additional_properties is None:<EOL><INDENT>return {}<EOL><DEDENT>merged_additional_parameters = deepcopy(additional_properties)<EOL>if '<STR_LIT>' not in merged_additional_parameters:<EOL><INDENT>merged_additional_parameters['<STR_LIT>'] = {}<EOL><DEDENT>merged_additional_parameters['<STR_LIT>']['<STR_LIT>'] = merged_additional_parameters[<EOL>'<STR_LIT>'<EOL>].get('<STR_LIT>', language_hints)<EOL>merged_additional_parameters['<STR_LIT>']['<STR_LIT>'] = merged_additional_parameters[<EOL>'<STR_LIT>'<EOL>].get('<STR_LIT>', web_detection_params)<EOL>return merged_additional_parameters<EOL>

Creates additional_properties parameter based on language_hints, web_detection_params and
additional_properties parameters specified by the user


try:<EOL><INDENT>waiter = self.client.get_waiter('<STR_LIT>')<EOL>waiter.config.max_attempts = sys.maxsize  <EOL>waiter.wait(jobs=[self.jobId])<EOL><DEDENT>except ValueError:<EOL><INDENT>retry = True<EOL>retries = <NUM_LIT:0><EOL>while retries < self.max_retries and retry:<EOL><INDENT>self.log.info('<STR_LIT>', retries)<EOL>response = self.client.describe_jobs(<EOL>jobs=[self.jobId]<EOL>)<EOL>if response['<STR_LIT>'][-<NUM_LIT:1>]['<STR_LIT:status>'] in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>retry = False<EOL><DEDENT>sleep(<NUM_LIT:1> + pow(retries * <NUM_LIT:0.1>, <NUM_LIT:2>))<EOL>retries += <NUM_LIT:1><EOL><DEDENT><DEDENT>

Try to use a waiter from the below pull request

    * https://github.com/boto/botocore/pull/1307

If the waiter is not available apply a exponential backoff

    * docs.aws.amazon.com/general/latest/gr/api-retries.html


super().__init__(**kwargs)<EOL>self.json = json or {}<EOL>self.databricks_conn_id = databricks_conn_id<EOL>self.polling_period_seconds = polling_period_seconds<EOL>self.databricks_retry_limit = databricks_retry_limit<EOL>self.databricks_retry_delay = databricks_retry_delay<EOL>if job_id is not None:<EOL><INDENT>self.json['<STR_LIT>'] = job_id<EOL><DEDENT>if notebook_params is not None:<EOL><INDENT>self.json['<STR_LIT>'] = notebook_params<EOL><DEDENT>if python_params is not None:<EOL><INDENT>self.json['<STR_LIT>'] = python_params<EOL><DEDENT>if spark_submit_params is not None:<EOL><INDENT>self.json['<STR_LIT>'] = spark_submit_params<EOL><DEDENT>self.json = _deep_string_coerce(self.json)<EOL>self.run_id = None<EOL>self.do_xcom_push = do_xcom_push<EOL>

Creates a new ``DatabricksRunNowOperator``.


self.log.info(<EOL>'<STR_LIT>',<EOL>self.imap_attachment_name, self.s3_key<EOL>)<EOL>with ImapHook(imap_conn_id=self.imap_conn_id) as imap_hook:<EOL><INDENT>imap_mail_attachments = imap_hook.retrieve_mail_attachments(<EOL>name=self.imap_attachment_name,<EOL>mail_folder=self.imap_mail_folder,<EOL>check_regex=self.imap_check_regex,<EOL>latest_only=True<EOL>)<EOL><DEDENT>s3_hook = S3Hook(aws_conn_id=self.s3_conn_id)<EOL>s3_hook.load_bytes(bytes_data=imap_mail_attachments[<NUM_LIT:0>][<NUM_LIT:1>], key=self.s3_key)<EOL>

This function executes the transfer from the email server (via imap) into s3.

:param context: The context while executing.
:type context: dict


if remote_directory == '<STR_LIT:/>':<EOL><INDENT>sftp_client.chdir('<STR_LIT:/>')<EOL>return<EOL><DEDENT>if remote_directory == '<STR_LIT>':<EOL><INDENT>return<EOL><DEDENT>try:<EOL><INDENT>sftp_client.chdir(remote_directory)<EOL><DEDENT>except IOError:<EOL><INDENT>dirname, basename = os.path.split(remote_directory.rstrip('<STR_LIT:/>'))<EOL>_make_intermediate_dirs(sftp_client, dirname)<EOL>sftp_client.mkdir(basename)<EOL>sftp_client.chdir(basename)<EOL>return<EOL><DEDENT>

Create all the intermediate directories in a remote host

:param sftp_client: A Paramiko SFTP client.
:param remote_directory: Absolute Path of the directory containing the file
:return:


if not file_name.startswith('<STR_LIT>'):<EOL><INDENT>return file_name<EOL><DEDENT>path_components = file_name[self.GCS_PREFIX_LENGTH:].split('<STR_LIT:/>')<EOL>if len(path_components) < <NUM_LIT:2>:<EOL><INDENT>raise Exception(<EOL>'<STR_LIT>'<EOL>.format(file_name))<EOL><DEDENT>bucket_id = path_components[<NUM_LIT:0>]<EOL>object_id = '<STR_LIT:/>'.join(path_components[<NUM_LIT:1>:])<EOL>local_file = '<STR_LIT>'.format(str(uuid.uuid4())[:<NUM_LIT:8>],<EOL>path_components[-<NUM_LIT:1>])<EOL>self._gcs_hook.download(bucket_id, object_id, local_file)<EOL>if os.stat(local_file).st_size > <NUM_LIT:0>:<EOL><INDENT>return local_file<EOL><DEDENT>raise Exception(<EOL>'<STR_LIT>'<EOL>.format(file_name))<EOL>

Checks whether the file specified by file_name is stored in Google Cloud
Storage (GCS), if so, downloads the file and saves it locally. The full
path of the saved file will be returned. Otherwise the local file_name
will be returned immediately.

:param file_name: The full path of input file.
:type file_name: str
:return: The full path of local file.
:rtype: str


<EOL>if self.parameters and isinstance(self.parameters, six.string_types):<EOL><INDENT>import ast<EOL>self.parameters = ast.literal_eval(self.parameters)<EOL><DEDENT>if not self.parameters:<EOL><INDENT>self.parameters = None<EOL><DEDENT>request = Request(jenkins_server.build_job_url(self.job_name,<EOL>self.parameters, None))<EOL>return jenkins_request_with_headers(jenkins_server, request)<EOL>

This function makes an API call to Jenkins to trigger a build for 'job_name'
It returned a dict with 2 keys : body and headers.
headers contains also a dict-like object which can be queried to get
the location to poll in the queue.

:param jenkins_server: The jenkins server where the job should be triggered
:return: Dict containing the response body (key body)
    and the headers coming along (headers)


def poll_job_in_queue(self, location, jenkins_server):

try_count = <NUM_LIT:0><EOL>location = location + '<STR_LIT>'<EOL>self.log.info('<STR_LIT>', location)<EOL>while try_count < self.max_try_before_job_appears:<EOL><INDENT>location_answer = jenkins_request_with_headers(jenkins_server,<EOL>Request(location))<EOL>if location_answer is not None:<EOL><INDENT>json_response = json.loads(location_answer['<STR_LIT:body>'])<EOL>if '<STR_LIT>' in json_response:<EOL><INDENT>build_number = json_response['<STR_LIT>']['<STR_LIT>']<EOL>self.log.info('<STR_LIT>',<EOL>build_number)<EOL>return build_number<EOL><DEDENT><DEDENT>try_count += <NUM_LIT:1><EOL>time.sleep(self.sleep_time)<EOL><DEDENT>raise AirflowException("<STR_LIT>"<EOL>"<STR_LIT>",<EOL>self.max_try_before_job_appears)<EOL>

This method poll the jenkins queue until the job is executed.
When we trigger a job through an API call,
the job is first put in the queue without having a build number assigned.
Thus we have to wait the job exit the queue to know its build number.
To do so, we have to add /api/json (or /api/xml) to the location
returned by the build_job call and poll this file.
When a 'executable' block appears in the json, it means the job execution started
and the field 'number' then contains the build number.

:param location: Location to poll, returned in the header of the build_job call
:param jenkins_server: The jenkins server to poll
:return: The build_number corresponding to the triggered job


parsed_s3_key = urlparse(s3_key)<EOL>return parsed_s3_key.path.lstrip('<STR_LIT:/>')<EOL>

This parses the correct format for S3 keys
            regardless of how the S3 url is passed.


def _configure_csv_file(self, file_handle, schema):

csv_writer = csv.writer(file_handle, encoding='<STR_LIT:utf-8>',<EOL>delimiter=self.field_delimiter)<EOL>csv_writer.writerow(schema)<EOL>return csv_writer<EOL>

Configure a csv writer with the file_handle and write schema
        as headers for the new file.


schema = []<EOL>if isinstance(self.schema, string_types):<EOL><INDENT>schema = json.loads(self.schema)<EOL><DEDENT>elif isinstance(self.schema, list):<EOL><INDENT>schema = self.schema<EOL><DEDENT>elif self.schema is not None:<EOL><INDENT>self.log.warn('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>col_type_dict = {}<EOL>try:<EOL><INDENT>col_type_dict = {col['<STR_LIT:name>']: col['<STR_LIT:type>'] for col in schema}<EOL><DEDENT>except KeyError:<EOL><INDENT>self.log.warn('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>return col_type_dict<EOL>

Return a dict of column name and column type based on self.schema if not None.


d = {<EOL><NUM_LIT:3>: '<STR_LIT>',<EOL><NUM_LIT:4>: '<STR_LIT>',<EOL><NUM_LIT:5>: '<STR_LIT>'<EOL>}<EOL>return d[mssql_type] if mssql_type in d else '<STR_LIT>'<EOL>

Helper function that maps from MSSQL fields to BigQuery fields. Used
when a schema_filename is set.


schema = []<EOL>for field in cursor.description:<EOL><INDENT>field_name = field[<NUM_LIT:0>].replace('<STR_LIT:U+0020>', '<STR_LIT:_>')  <EOL>field_type = self.type_map(field[<NUM_LIT:1>])<EOL>field_mode = '<STR_LIT>'  <EOL>schema.append({<EOL>'<STR_LIT:name>': field_name,<EOL>'<STR_LIT:type>': field_type,<EOL>'<STR_LIT>': field_mode,<EOL>})<EOL><DEDENT>self.log.info('<STR_LIT>', self.schema_filename, schema)<EOL>tmp_schema_file_handle = NamedTemporaryFile(delete=True)<EOL>s = json.dumps(schema, sort_keys=True)<EOL>s = s.encode('<STR_LIT:utf-8>')<EOL>tmp_schema_file_handle.write(s)<EOL>return {self.schema_filename: tmp_schema_file_handle}<EOL>

Takes a cursor, and writes the BigQuery schema for the results to a
local file system.

:return: A dictionary where key is a filename to be used as an object
    name in GCS, and values are file handles to local files that
    contains the BigQuery schema fields in .json format.


Call the SparkSubmitHook to run the provided spark job


self.hook = CassandraHook(cassandra_conn_id=self.cassandra_conn_id)<EOL>session = self.hook.get_conn()<EOL>cursor = session.execute(self.cql)<EOL>return cursor<EOL>

Queries cassandra and returns a cursor to the results.


return self.data_profiler<EOL>


return True<EOL>


return False<EOL>


return Response("<STR_LIT>", <NUM_LIT>, {"<STR_LIT>": "<STR_LIT>"})<EOL>

Indicate that authorization is required
:return:


raise NotImplementedError()<EOL>

:return: The return code associated with running the task instance or
    None if the task is not yet done.
:rtype: int


return getattr(self, '<STR_LIT>', self.__class__.__name__)<EOL>

The human-readable name for the dependency. Use the classname as the default name
if this method is not overridden in the subclass.


connectable = settings.engine<EOL>with connectable.connect() as connection:<EOL><INDENT>context.configure(<EOL>connection=connection,<EOL>transaction_per_migration=True,<EOL>target_metadata=target_metadata,<EOL>compare_type=COMPARE_TYPE,<EOL>)<EOL>with context.begin_transaction():<EOL><INDENT>context.run_migrations()<EOL><DEDENT><DEDENT>

Run migrations in 'online' mode.

    In this scenario we need to create an Engine
    and associate a connection with the context.


Run migrations in 'offline' mode.

    This configures the context with just a URL
    and not an Engine, though an Engine is acceptable
    here as well.  By skipping the Engine creation
    we don't even need a DBAPI to be available.

    Calls to context.execute() here emit the given string to the
    script output.


def sync_perm_for_dag(self, dag_id, access_control=None):

for dag_perm in self.DAG_PERMS:<EOL><INDENT>perm_on_dag = self.find_permission_view_menu(dag_perm, dag_id)<EOL>if perm_on_dag is None:<EOL><INDENT>self.add_permission_view_menu(dag_perm, dag_id)<EOL><DEDENT><DEDENT>if access_control:<EOL><INDENT>self._sync_dag_view_permissions(dag_id, access_control)<EOL><DEDENT>

Sync permissions for given dag id. The dag id surely exists in our dag bag
as only / refresh button or cli.sync_perm will call this function

:param dag_id: the ID of the DAG whose permissions should be updated
:type dag_id: string
:param access_control: a dict where each key is a rolename and
    each value is a set() of permission names (e.g.,
    {'can_dag_read'}
:type access_control: dict
:return:


def _has_perm(self, permission_name, view_menu_name):

if hasattr(self, '<STR_LIT>'):<EOL><INDENT>if (permission_name, view_menu_name) in self.perms:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>self._get_and_cache_perms()<EOL>return (permission_name, view_menu_name) in self.perms<EOL>


if not username:<EOL><INDENT>username = g.user<EOL><DEDENT>if username.is_anonymous or '<STR_LIT>' in username.roles:<EOL><INDENT>return set()<EOL><DEDENT>roles = {role.name for role in username.roles}<EOL>if {'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'} & roles:<EOL><INDENT>return self.DAG_VMS<EOL><DEDENT>user_perms_views = self.get_all_permissions_views()<EOL>return set([view for perm, view in user_perms_views if perm in self.DAG_PERMS])<EOL>

Return a set of dags that user has access to(either read or write).

:param username: Name of the user.
:return: A set of dag ids that the user could access.


return (<EOL>self._has_role(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']) or<EOL>self._has_perm('<STR_LIT>', '<STR_LIT>') or<EOL>self._has_perm('<STR_LIT>', '<STR_LIT>'))<EOL>

Has all the dag access in any of the 3 cases:
1. Role needs to be in (Admin, Viewer, User, Op).
2. Has can_dag_read permission on all_dags view.
3. Has can_dag_edit permission on all_dags view.


perms_views = set()<EOL>for role in self.get_user_roles():<EOL><INDENT>perms_views.update({(perm_view.permission.name, perm_view.view_menu.name)<EOL>for perm_view in role.permissions})<EOL><DEDENT>return perms_views<EOL>

Returns a set of tuples with the perm name and view menu name


pvms = self.get_session.query(sqla_models.PermissionView).all()<EOL>pvms = [p for p in pvms if p.permission and p.view_menu]<EOL>admin = self.find_role('<STR_LIT>')<EOL>admin.permissions = list(set(admin.permissions) | set(pvms))<EOL>self.get_session.commit()<EOL>

Admin should have all the permission-views.
Add the missing ones to the table for admin.

:return: None.


if user is None:<EOL><INDENT>user = g.user<EOL><DEDENT>if user.is_anonymous:<EOL><INDENT>public_role = appbuilder.config.get('<STR_LIT>')<EOL>return [appbuilder.security_manager.find_role(public_role)]if public_role else []<EOL><DEDENT>return user.roles<EOL>

Get all the roles associated with the user.

:param user: the ab_user in FAB model.
:return: a list of roles associated with the user.


self.log.debug('<STR_LIT>')<EOL>self.create_perm_vm_for_all_dag()<EOL>for config in self.ROLE_CONFIGS:<EOL><INDENT>role = config['<STR_LIT>']<EOL>vms = config['<STR_LIT>']<EOL>perms = config['<STR_LIT>']<EOL>self.init_role(role, vms, perms)<EOL><DEDENT>self.create_custom_dag_permission_view()<EOL>self.update_admin_perm_view()<EOL>self.clean_perms()<EOL>

1. Init the default role(Admin, Viewer, User, Op, public)
   with related permissions.
2. Init the custom role(dag-user) with related permissions.

:return: None.


try:<EOL><INDENT>info = get_task(dag_id, task_id)<EOL><DEDENT>except AirflowException as err:<EOL><INDENT>_log.info(err)<EOL>response = jsonify(error="<STR_LIT:{}>".format(err))<EOL>response.status_code = err.status_code<EOL>return response<EOL><DEDENT>fields = {k: str(v)<EOL>for k, v in vars(info).items()<EOL>if not k.startswith('<STR_LIT:_>')}<EOL>return jsonify(fields)<EOL>

Returns a JSON with a task's public instance variables.


try:<EOL><INDENT>state = request.args.get('<STR_LIT:state>')<EOL>dagruns = get_dag_runs(dag_id, state)<EOL><DEDENT>except AirflowException as err:<EOL><INDENT>_log.info(err)<EOL>response = jsonify(error="<STR_LIT:{}>".format(err))<EOL>response.status_code = <NUM_LIT><EOL>return response<EOL><DEDENT>return jsonify(dagruns)<EOL>

Returns a list of Dag Runs for a specific DAG ID.
:query param state: a query string parameter '?state=queued|running|success...'
:param dag_id: String identifier of a DAG
:return: List of DAG runs of a DAG with requested state,
or all runs if the state is not specified


try:<EOL><INDENT>count = delete.delete_dag(dag_id)<EOL><DEDENT>except AirflowException as err:<EOL><INDENT>_log.error(err)<EOL>response = jsonify(error="<STR_LIT:{}>".format(err))<EOL>response.status_code = err.status_code<EOL>return response<EOL><DEDENT>return jsonify(message="<STR_LIT>".format(count), count=count)<EOL>

Delete all DB records related to the specified Dag.


<EOL>try:<EOL><INDENT>execution_date = timezone.parse(execution_date)<EOL><DEDENT>except ValueError:<EOL><INDENT>error_message = (<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'.format(<EOL>execution_date))<EOL>_log.info(error_message)<EOL>response = jsonify({'<STR_LIT:error>': error_message})<EOL>response.status_code = <NUM_LIT><EOL>return response<EOL><DEDENT>try:<EOL><INDENT>info = get_dag_run_state(dag_id, execution_date)<EOL><DEDENT>except AirflowException as err:<EOL><INDENT>_log.info(err)<EOL>response = jsonify(error="<STR_LIT:{}>".format(err))<EOL>response.status_code = err.status_code<EOL>return response<EOL><DEDENT>return jsonify(info)<EOL>

Returns a JSON with a dag_run's public instance variables.
The format for the exec_date is expected to be
"YYYY-mm-DDTHH:MM:SS", for example: "2016-11-16T11:34:15". This will
of course need to have been encoded for URL in the request.


DagModel = models.DagModel<EOL>with create_session() as session:<EOL><INDENT>orm_dag = (<EOL>session.query(DagModel)<EOL>.filter(DagModel.dag_id == dag_id).first()<EOL>)<EOL>if paused == '<STR_LIT:true>':<EOL><INDENT>orm_dag.is_paused = True<EOL><DEDENT>else:<EOL><INDENT>orm_dag.is_paused = False<EOL><DEDENT>session.merge(orm_dag)<EOL>session.commit()<EOL><DEDENT>return jsonify({'<STR_LIT>': '<STR_LIT>'})<EOL>


path = request.path<EOL>args = str(hash(frozenset(request.args.items())))<EOL>return (path + args).encode('<STR_LIT:ascii>', '<STR_LIT:ignore>')<EOL>

Used by cache to get a unique key per URL


return <NUM_LIT> + len(dag.tasks) * <NUM_LIT:10><EOL>

TODO(aoen): See [AIRFLOW-1263] We use the number of tasks in the DAG as a heuristic to
approximate the size of generated chart (otherwise the charts are tiny and unreadable
when DAGs have a large number of tasks). Ideally nvd3 should allow for dynamic-height
charts, that is charts that take up space based on the size of the components within.


return int(time.mktime(dttm.timetuple())) * <NUM_LIT:1000>,<EOL>


@functools.wraps(f)<EOL>def view_func(*args, **kwargs):<EOL><INDENT>@after_this_request<EOL>def zipper(response):<EOL><INDENT>accept_encoding = request.headers.get('<STR_LIT>', '<STR_LIT>')<EOL>if '<STR_LIT>' not in accept_encoding.lower():<EOL><INDENT>return response<EOL><DEDENT>response.direct_passthrough = False<EOL>if (response.status_code < <NUM_LIT:200> or response.status_code >= <NUM_LIT> or<EOL>'<STR_LIT>' in response.headers):<EOL><INDENT>return response<EOL><DEDENT>gzip_buffer = IO()<EOL>gzip_file = gzip.GzipFile(mode='<STR_LIT:wb>',<EOL>fileobj=gzip_buffer)<EOL>gzip_file.write(response.data)<EOL>gzip_file.close()<EOL>response.data = gzip_buffer.getvalue()<EOL>response.headers['<STR_LIT>'] = '<STR_LIT>'<EOL>response.headers['<STR_LIT>'] = '<STR_LIT>'<EOL>response.headers['<STR_LIT>'] = len(response.data)<EOL>return response<EOL><DEDENT>return f(*args, **kwargs)<EOL><DEDENT>return view_func<EOL>


def parse_manifest_json():<EOL><INDENT>try:<EOL><INDENT>global manifest<EOL>manifest_file = os.path.join(os.path.dirname(__file__),<EOL>'<STR_LIT>')<EOL>with open(manifest_file, '<STR_LIT:r>') as f:<EOL><INDENT>manifest.update(json.load(f))<EOL>for k in manifest.keys():<EOL><INDENT>manifest[k] = os.path.join("<STR_LIT>", manifest[k])<EOL><DEDENT><DEDENT><DEDENT>except Exception:<EOL><INDENT>print("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>pass<EOL><DEDENT><DEDENT>def get_asset_url(filename):<EOL><INDENT>if app.debug:<EOL><INDENT>parse_manifest_json()<EOL><DEDENT>return url_for('<STR_LIT>', filename=manifest.get(filename, '<STR_LIT>'))<EOL><DEDENT>parse_manifest_json()<EOL>@app.context_processor<EOL>def get_url_for_asset():<EOL><INDENT>"""<STR_LIT>"""<EOL>return dict(url_for_asset=get_asset_url)<EOL><DEDENT>

Loads the manifest file and register the `url_for_asset_` template tag.
:param app:
:return:


TI = TaskInstance<EOL>qry = session.query(TI).filter(TI.dag_id == self.dag_id)<EOL>if start_date:<EOL><INDENT>qry = qry.filter(TI.execution_date >= start_date)<EOL><DEDENT>if end_date:<EOL><INDENT>qry = qry.filter(TI.execution_date <= end_date)<EOL><DEDENT>tasks = [self.task_id]<EOL>if upstream:<EOL><INDENT>tasks += [<EOL>t.task_id for t in self.get_flat_relatives(upstream=True)]<EOL><DEDENT>if downstream:<EOL><INDENT>tasks += [<EOL>t.task_id for t in self.get_flat_relatives(upstream=False)]<EOL><DEDENT>qry = qry.filter(TI.task_id.in_(tasks))<EOL>count = qry.count()<EOL>clear_task_instances(qry.all(), session, dag=self.dag)<EOL>session.commit()<EOL>return count<EOL>

Clears the state of task instances associated with the task, following
the parameters specified.


return list(map(lambda task_id: self._dag.task_dict[task_id],<EOL>self.get_flat_relative_ids(upstream)))<EOL>

Get a flat list of relatives, either upstream or downstream.


return [self.dag.get_task(tid) for tid in self._upstream_task_ids]<EOL>

@property: list of tasks directly upstream


pass<EOL>

Override this method to cleanup subprocesses when a task instance
gets killed. Any use of the threading, subprocess or multiprocessing
module within an operator needs to be cleaned up or it will leave
ghost processes behind.


def get_flat_relative_ids(self, upstream=False, found_descendants=None):

if not found_descendants:<EOL><INDENT>found_descendants = set()<EOL><DEDENT>relative_ids = self.get_direct_relative_ids(upstream)<EOL>for relative_id in relative_ids:<EOL><INDENT>if relative_id not in found_descendants:<EOL><INDENT>found_descendants.add(relative_id)<EOL>relative_task = self._dag.task_dict[relative_id]<EOL>relative_task.get_flat_relative_ids(upstream,<EOL>found_descendants)<EOL><DEDENT><DEDENT>return found_descendants<EOL>

Get a flat list of relatives' ids, either upstream or downstream.


@provide_session<EOL><INDENT>def get_task_instances(self, start_date=None, end_date=None, session=None):<DEDENT>

end_date = end_date or timezone.utcnow()<EOL>return session.query(TaskInstance).filter(TaskInstance.dag_id == self.dag_id).filter(TaskInstance.task_id == self.task_id).filter(TaskInstance.execution_date >= start_date).filter(TaskInstance.execution_date <= end_date).order_by(TaskInstance.execution_date).all()<EOL>

Get a set of task instance related to this task for a specific date
range.


@abstractmethod<EOL><INDENT>def get_link(self, operator, dttm):<EOL><DEDENT>

pass<EOL>

Link to external system.

:param operator: airflow operator
:param dttm: datetime
:return: link to external system


raise NotImplementedError()<EOL>

This is the main method to derive when creating an operator.
Context is the same dictionary used as when rendering jinja templates.

Refer to get_template_context for more context.


filters = []<EOL>if key:<EOL><INDENT>filters.append(cls.key == key)<EOL><DEDENT>if task_ids:<EOL><INDENT>filters.append(cls.task_id.in_(as_tuple(task_ids)))<EOL><DEDENT>if dag_ids:<EOL><INDENT>filters.append(cls.dag_id.in_(as_tuple(dag_ids)))<EOL><DEDENT>if include_prior_dates:<EOL><INDENT>filters.append(cls.execution_date <= execution_date)<EOL><DEDENT>else:<EOL><INDENT>filters.append(cls.execution_date == execution_date)<EOL><DEDENT>query = (<EOL>session.query(cls).filter(and_(*filters))<EOL>.order_by(cls.execution_date.desc(), cls.timestamp.desc())<EOL>.limit(limit))<EOL>results = query.all()<EOL>return results<EOL>

Retrieve an XCom value, optionally meeting certain criteria
TODO: "pickling" has been deprecated and JSON is preferred.
"pickling" will be removed in Airflow 2.0.


filters = []<EOL>if key:<EOL><INDENT>filters.append(cls.key == key)<EOL><DEDENT>if task_id:<EOL><INDENT>filters.append(cls.task_id == task_id)<EOL><DEDENT>if dag_id:<EOL><INDENT>filters.append(cls.dag_id == dag_id)<EOL><DEDENT>if include_prior_dates:<EOL><INDENT>filters.append(cls.execution_date <= execution_date)<EOL><DEDENT>else:<EOL><INDENT>filters.append(cls.execution_date == execution_date)<EOL><DEDENT>query = (<EOL>session.query(cls.value).filter(and_(*filters))<EOL>.order_by(cls.execution_date.desc(), cls.timestamp.desc()))<EOL>result = query.first()<EOL>if result:<EOL><INDENT>enable_pickling = configuration.getboolean('<STR_LIT>', '<STR_LIT>')<EOL>if enable_pickling:<EOL><INDENT>return pickle.loads(result.value)<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>return json.loads(result.value.decode('<STR_LIT>'))<EOL><DEDENT>except ValueError:<EOL><INDENT>log = LoggingMixin().log<EOL>log.error("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL>raise<EOL><DEDENT><DEDENT><DEDENT>

Retrieve an XCom value, optionally meeting certain criteria.
TODO: "pickling" has been deprecated and JSON is preferred.
"pickling" will be removed in Airflow 2.0.

:return: XCom value


obj = {}<EOL>if self.extra:<EOL><INDENT>try:<EOL><INDENT>obj = json.loads(self.extra)<EOL><DEDENT>except Exception as e:<EOL><INDENT>self.log.exception(e)<EOL>self.log.error("<STR_LIT>", self.conn_id)<EOL><DEDENT><DEDENT>return obj<EOL>

Returns the extra property by deserializing json.


@provide_session<EOL><INDENT>def sync_to_db(self, owner=None, sync_time=None, session=None):<DEDENT>

if owner is None:<EOL><INDENT>owner = self.owner<EOL><DEDENT>if sync_time is None:<EOL><INDENT>sync_time = timezone.utcnow()<EOL><DEDENT>orm_dag = session.query(<EOL>DagModel).filter(DagModel.dag_id == self.dag_id).first()<EOL>if not orm_dag:<EOL><INDENT>orm_dag = DagModel(dag_id=self.dag_id)<EOL>self.log.info("<STR_LIT>", self.dag_id)<EOL><DEDENT>orm_dag.fileloc = self.fileloc<EOL>orm_dag.is_subdag = self.is_subdag<EOL>orm_dag.owners = owner<EOL>orm_dag.is_active = True<EOL>orm_dag.last_scheduler_run = sync_time<EOL>orm_dag.default_view = self._default_view<EOL>orm_dag.description = self.description<EOL>orm_dag.schedule_interval = self.schedule_interval<EOL>session.merge(orm_dag)<EOL>session.commit()<EOL>for subdag in self.subdags:<EOL><INDENT>subdag.sync_to_db(owner=owner, sync_time=sync_time, session=session)<EOL><DEDENT>

Save attributes about this DAG to the DB. Note that this method
can be called for both DAGs and SubDAGs. A SubDag is actually a
SubDagOperator.

:param dag: the DAG object to save to the DB
:type dag: airflow.models.DAG
:param sync_time: The time that the DAG should be marked as sync'ed
:type sync_time: datetime
:return: None


runs = DagRun.find(dag_id=self.dag_id, state=State.RUNNING)<EOL>active_dates = []<EOL>for run in runs:<EOL><INDENT>active_dates.append(run.execution_date)<EOL><DEDENT>return active_dates<EOL>

Returns a list of dag run execution dates currently running

:return: List of execution dates


for task in tasks:<EOL><INDENT>self.add_task(task)<EOL><DEDENT>

Add a list of tasks to the DAG

:param tasks: a lit of tasks you want to add
:type tasks: list of tasks


<EOL>graph_unsorted = OrderedDict((task.task_id, task) for task in self.tasks)<EOL>graph_sorted = []<EOL>if len(self.tasks) == <NUM_LIT:0>:<EOL><INDENT>return tuple(graph_sorted)<EOL><DEDENT>while graph_unsorted:<EOL><INDENT>acyclic = False<EOL>for node in list(graph_unsorted.values()):<EOL><INDENT>for edge in node.upstream_list:<EOL><INDENT>if edge.task_id in graph_unsorted:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>acyclic = True<EOL>del graph_unsorted[node.task_id]<EOL>graph_sorted.append(node)<EOL><DEDENT><DEDENT>if not acyclic:<EOL><INDENT>raise AirflowException("<STR_LIT>"<EOL>.format(self.dag_id))<EOL><DEDENT><DEDENT>return tuple(graph_sorted)<EOL>

Sorts tasks in topographical order, such that a task comes after any of its
upstream dependencies.

Heavily inspired by:
http://blog.jupo.org/2012/04/06/topological-sorting-acyclic-directed-graphs/

:return: list of tasks in topological order


from airflow.jobs import BackfillJob<EOL>if not executor and local:<EOL><INDENT>executor = LocalExecutor()<EOL><DEDENT>elif not executor:<EOL><INDENT>executor = get_default_executor()<EOL><DEDENT>job = BackfillJob(<EOL>self,<EOL>start_date=start_date,<EOL>end_date=end_date,<EOL>mark_success=mark_success,<EOL>executor=executor,<EOL>donot_pickle=donot_pickle,<EOL>ignore_task_deps=ignore_task_deps,<EOL>ignore_first_depends_on_past=ignore_first_depends_on_past,<EOL>pool=pool,<EOL>delay_on_limit_secs=delay_on_limit_secs,<EOL>verbose=verbose,<EOL>conf=conf,<EOL>rerun_failed_tasks=rerun_failed_tasks,<EOL>run_backwards=run_backwards,<EOL>)<EOL>job.run()<EOL>

Runs the DAG.

:param start_date: the start date of the range to run
:type start_date: datetime.datetime
:param end_date: the end date of the range to run
:type end_date: datetime.datetime
:param mark_success: True to mark jobs as succeeded without running them
:type mark_success: bool
:param local: True to run the tasks using the LocalExecutor
:type local: bool
:param executor: The executor instance to run the tasks
:type executor: airflow.executor.BaseExecutor
:param donot_pickle: True to avoid pickling DAG object and send to workers
:type donot_pickle: bool
:param ignore_task_deps: True to skip upstream tasks
:type ignore_task_deps: bool
:param ignore_first_depends_on_past: True to ignore depends_on_past
    dependencies for the first set of tasks only
:type ignore_first_depends_on_past: bool
:param pool: Resource pool to use
:type pool: str
:param delay_on_limit_secs: Time in seconds to wait before next attempt to run
    dag run when max_active_runs limit has been reached
:type delay_on_limit_secs: float
:param verbose: Make logging output more verbose
:type verbose: bool
:param conf: user defined dictionary passed from CLI
:type conf: dict
:param rerun_failed_tasks:
:type: bool
:param run_backwards:
:type: bool


return os.path.dirname(self.full_filepath)<EOL>

Folder location of where the dag object is instantiated


@provide_session<EOL><INDENT>def skip(self, dag_run, execution_date, tasks, session=None):<DEDENT>

if not tasks:<EOL><INDENT>return<EOL><DEDENT>task_ids = [d.task_id for d in tasks]<EOL>now = timezone.utcnow()<EOL>if dag_run:<EOL><INDENT>session.query(TaskInstance).filter(<EOL>TaskInstance.dag_id == dag_run.dag_id,<EOL>TaskInstance.execution_date == dag_run.execution_date,<EOL>TaskInstance.task_id.in_(task_ids)<EOL>).update({TaskInstance.state: State.SKIPPED,<EOL>TaskInstance.start_date: now,<EOL>TaskInstance.end_date: now},<EOL>synchronize_session=False)<EOL>session.commit()<EOL><DEDENT>else:<EOL><INDENT>assert execution_date is not None, "<STR_LIT>"<EOL>self.log.warning("<STR_LIT>")<EOL>for task in tasks:<EOL><INDENT>ti = TaskInstance(task, execution_date=execution_date)<EOL>ti.state = State.SKIPPED<EOL>ti.start_date = now<EOL>ti.end_date = now<EOL>session.merge(ti)<EOL><DEDENT>session.commit()<EOL><DEDENT>

Sets tasks instances to skipped from the same dag run.

:param dag_run: the DagRun for which to set the tasks to skipped
:param execution_date: execution_date
:param tasks: tasks to skip (not task_ids)
:param session: db session to use


return self.task.retries and self.try_number <= self.max_tries<EOL>


@provide_session<EOL><INDENT>def refresh_from_db(self, session=None, lock_for_update=False):<DEDENT>

TI = TaskInstance<EOL>qry = session.query(TI).filter(<EOL>TI.dag_id == self.dag_id,<EOL>TI.task_id == self.task_id,<EOL>TI.execution_date == self.execution_date)<EOL>if lock_for_update:<EOL><INDENT>ti = qry.with_for_update().first()<EOL><DEDENT>else:<EOL><INDENT>ti = qry.first()<EOL><DEDENT>if ti:<EOL><INDENT>self.state = ti.state<EOL>self.start_date = ti.start_date<EOL>self.end_date = ti.end_date<EOL>self.try_number = ti._try_number<EOL>self.max_tries = ti.max_tries<EOL>self.hostname = ti.hostname<EOL>self.pid = ti.pid<EOL>self.executor_config = ti.executor_config<EOL><DEDENT>else:<EOL><INDENT>self.state = None<EOL><DEDENT>

Refreshes the task instance from the database based on the primary key

:param lock_for_update: if True, indicates that the database should
    lock the TaskInstance (issuing a FOR UPDATE clause) until the
    session is committed.


return (self.state == State.UP_FOR_RETRY and<EOL>self.next_retry_datetime() < timezone.utcnow())<EOL>

Checks on whether the task instance is in the right state and timeframe
to be retried.


from airflow.models.dagrun import DagRun  <EOL>dr = session.query(DagRun).filter(<EOL>DagRun.dag_id == self.dag_id,<EOL>DagRun.execution_date == self.execution_date<EOL>).first()<EOL>return dr<EOL>

Returns the DagRun for this TaskInstance

:param session:
:return: DagRun


session.query(XCom).filter(<EOL>XCom.dag_id == self.dag_id,<EOL>XCom.task_id == self.task_id,<EOL>XCom.execution_date == self.execution_date<EOL>).delete()<EOL>session.commit()<EOL>

Clears all XCom data from the database for the task instance


dep_context = dep_context or DepContext()<EOL>failed = False<EOL>verbose_aware_logger = self.log.info if verbose else self.log.debug<EOL>for dep_status in self.get_failed_dep_statuses(<EOL>dep_context=dep_context,<EOL>session=session):<EOL><INDENT>failed = True<EOL>verbose_aware_logger(<EOL>"<STR_LIT>",<EOL>self, dep_status.dep_name, dep_status.reason<EOL>)<EOL><DEDENT>if failed:<EOL><INDENT>return False<EOL><DEDENT>verbose_aware_logger("<STR_LIT>", self)<EOL>return True<EOL>

Returns whether or not all the conditions are met for this task instance to be run
given the context for the dependencies (e.g. a task instance being force run from
the UI will ignore some dependencies).

:param dep_context: The execution context that determines the dependencies that
    should be evaluated.
:type dep_context: DepContext
:param session: database session
:type session: sqlalchemy.orm.session.Session
:param verbose: whether log details on failed dependencies on
    info or debug log level
:type verbose: bool


@staticmethod<EOL><INDENT>@provide_session<EOL>def find(dag_id=None, run_id=None, execution_date=None,<EOL>state=None, external_trigger=None, no_backfills=False,<EOL>session=None):<DEDENT>

DR = DagRun<EOL>qry = session.query(DR)<EOL>if dag_id:<EOL><INDENT>qry = qry.filter(DR.dag_id == dag_id)<EOL><DEDENT>if run_id:<EOL><INDENT>qry = qry.filter(DR.run_id == run_id)<EOL><DEDENT>if execution_date:<EOL><INDENT>if isinstance(execution_date, list):<EOL><INDENT>qry = qry.filter(DR.execution_date.in_(execution_date))<EOL><DEDENT>else:<EOL><INDENT>qry = qry.filter(DR.execution_date == execution_date)<EOL><DEDENT><DEDENT>if state:<EOL><INDENT>qry = qry.filter(DR.state == state)<EOL><DEDENT>if external_trigger is not None:<EOL><INDENT>qry = qry.filter(DR.external_trigger == external_trigger)<EOL><DEDENT>if no_backfills:<EOL><INDENT>from airflow.jobs import BackfillJob<EOL>qry = qry.filter(DR.run_id.notlike(BackfillJob.ID_PREFIX + '<STR_LIT:%>'))<EOL><DEDENT>dr = qry.order_by(DR.execution_date).all()<EOL>return dr<EOL>

Returns a set of dag runs for the given search criteria.

:param dag_id: the dag_id to find dag runs for
:type dag_id: int, list
:param run_id: defines the the run id for this dag run
:type run_id: str
:param execution_date: the execution date
:type execution_date: datetime.datetime
:param state: the state of the dag run
:type state: airflow.utils.state.State
:param external_trigger: whether this dag run is externally triggered
:type external_trigger: bool
:param no_backfills: return no backfills (True), return all (False).
    Defaults to False
:type no_backfills: bool
:param session: database session
:type session: sqlalchemy.orm.session.Session


if not self.dag:<EOL><INDENT>raise AirflowException("<STR_LIT>"<EOL>.format(self))<EOL><DEDENT>return self.dag<EOL>

Returns the Dag associated with this DagRun.

:return: DAG


@staticmethod<EOL><INDENT>def get_run(session, dag_id, execution_date):<DEDENT>

qry = session.query(DagRun).filter(<EOL>DagRun.dag_id == dag_id,<EOL>DagRun.external_trigger == False, <EOL>DagRun.execution_date == execution_date,<EOL>)<EOL>return qry.first()<EOL>

:param dag_id: DAG ID
:type dag_id: unicode
:param execution_date: execution date
:type execution_date: datetime
:return: DagRun corresponding to the given dag_id and execution date
    if one exists. None otherwise.
:rtype: airflow.models.DagRun


@provide_session<EOL><INDENT>def get_task_instance(self, task_id, session=None):<DEDENT>

from airflow.models.taskinstance import TaskInstance  <EOL>TI = TaskInstance<EOL>ti = session.query(TI).filter(<EOL>TI.dag_id == self.dag_id,<EOL>TI.execution_date == self.execution_date,<EOL>TI.task_id == task_id<EOL>).first()<EOL>return ti<EOL>

Returns the task instance specified by task_id for this dag run

:param task_id: the task id


from airflow.models.taskinstance import TaskInstance  <EOL>dag = self.get_dag()<EOL>tis = self.get_task_instances(session=session)<EOL>task_ids = []<EOL>for ti in tis:<EOL><INDENT>task_ids.append(ti.task_id)<EOL>task = None<EOL>try:<EOL><INDENT>task = dag.get_task(ti.task_id)<EOL><DEDENT>except AirflowException:<EOL><INDENT>if ti.state == State.REMOVED:<EOL><INDENT>pass  <EOL><DEDENT>elif self.state is not State.RUNNING and not dag.partial:<EOL><INDENT>self.log.warning("<STR_LIT>"<EOL>"<STR_LIT>".format(ti, dag))<EOL>Stats.incr(<EOL>"<STR_LIT>".format(dag.dag_id), <NUM_LIT:1>, <NUM_LIT:1>)<EOL>ti.state = State.REMOVED<EOL><DEDENT><DEDENT>is_task_in_dag = task is not None<EOL>should_restore_task = is_task_in_dag and ti.state == State.REMOVED<EOL>if should_restore_task:<EOL><INDENT>self.log.info("<STR_LIT>"<EOL>"<STR_LIT>".format(ti, dag))<EOL>Stats.incr("<STR_LIT>".format(dag.dag_id), <NUM_LIT:1>, <NUM_LIT:1>)<EOL>ti.state = State.NONE<EOL><DEDENT><DEDENT>for task in six.itervalues(dag.task_dict):<EOL><INDENT>if task.start_date > self.execution_date and not self.is_backfill:<EOL><INDENT>continue<EOL><DEDENT>if task.task_id not in task_ids:<EOL><INDENT>Stats.incr(<EOL>"<STR_LIT>".format(task.__class__.__name__),<EOL><NUM_LIT:1>, <NUM_LIT:1>)<EOL>ti = TaskInstance(task, self.execution_date)<EOL>session.add(ti)<EOL><DEDENT><DEDENT>session.commit()<EOL>

Verifies the DagRun by checking for removed tasks or tasks that are not in the
database yet. It will set state to removed or add the task if required.


@provide_session<EOL><INDENT>def _task_instances_for_dag_run(self, dag_run, session=None):<DEDENT>

tasks_to_run = {}<EOL>if dag_run is None:<EOL><INDENT>return tasks_to_run<EOL><DEDENT>self.reset_state_for_orphaned_tasks(filter_by_dag_run=dag_run, session=session)<EOL>dag_run.refresh_from_db()<EOL>make_transient(dag_run)<EOL>for ti in dag_run.get_task_instances():<EOL><INDENT>if ti.state == State.NONE:<EOL><INDENT>ti.set_state(State.SCHEDULED, session=session)<EOL><DEDENT>if ti.state != State.REMOVED:<EOL><INDENT>tasks_to_run[ti.key] = ti<EOL><DEDENT><DEDENT>return tasks_to_run<EOL>

Returns a map of task instance key to task instance object for the tasks to
run in the given dag run.

:param dag_run: the dag run to get the tasks from
:type dag_run: airflow.models.DagRun
:param session: the database session object
:type session: sqlalchemy.orm.session.Session


if self._start_time is None:<EOL><INDENT>raise AirflowException("<STR_LIT>")<EOL><DEDENT>return self._start_time<EOL>

:return: when this started to process the file
:rtype: datetime


@provide_session<EOL><INDENT>def _set_unfinished_dag_runs_to_failed(self, dag_runs, session=None):<DEDENT>

for dag_run in dag_runs:<EOL><INDENT>dag_run.update_state()<EOL>if dag_run.state not in State.finished():<EOL><INDENT>dag_run.set_state(State.FAILED)<EOL><DEDENT>session.merge(dag_run)<EOL><DEDENT>

Go through the dag_runs and update the state based on the task_instance state.
Then set DAG runs that are not finished to failed.

:param dag_runs: DAG runs
:param session: session
:return: None


if not self.done:<EOL><INDENT>raise AirflowException("<STR_LIT>")<EOL><DEDENT>return self._result<EOL>

:return: result of running SchedulerJob.process_file()
:rtype: airflow.utils.dag_processing.SimpleDag


executed_run_dates = []<EOL>while ((len(ti_status.to_run) > <NUM_LIT:0> or len(ti_status.running) > <NUM_LIT:0>) and<EOL>len(ti_status.deadlocked) == <NUM_LIT:0>):<EOL><INDENT>self.log.debug("<STR_LIT>")<EOL>ti_status.not_ready.clear()<EOL>@provide_session<EOL>def _per_task_process(task, key, ti, session=None):<EOL><INDENT>ti.refresh_from_db()<EOL>task = self.dag.get_task(ti.task_id)<EOL>ti.task = task<EOL>ignore_depends_on_past = (<EOL>self.ignore_first_depends_on_past and<EOL>ti.execution_date == (start_date or ti.start_date))<EOL>self.log.debug(<EOL>"<STR_LIT>", ti, ti.state)<EOL>if ti.state == State.SUCCESS:<EOL><INDENT>ti_status.succeeded.add(key)<EOL>self.log.debug("<STR_LIT>", ti)<EOL>ti_status.to_run.pop(key)<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>return<EOL><DEDENT>elif ti.state == State.SKIPPED:<EOL><INDENT>ti_status.skipped.add(key)<EOL>self.log.debug("<STR_LIT>", ti)<EOL>ti_status.to_run.pop(key)<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>return<EOL><DEDENT>elif ti.state == State.NONE:<EOL><INDENT>self.log.warning(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL>ti.set_state(State.SCHEDULED, session=session)<EOL><DEDENT>if self.rerun_failed_tasks:<EOL><INDENT>if ti.state in (State.FAILED, State.UPSTREAM_FAILED):<EOL><INDENT>self.log.error("<STR_LIT>"<EOL>"<STR_LIT>".format(ti=ti,<EOL>state=ti.state))<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>ti.set_state(State.SCHEDULED, session=session)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if ti.state in (State.FAILED, State.UPSTREAM_FAILED):<EOL><INDENT>self.log.error("<STR_LIT>"<EOL>"<STR_LIT>".format(ti=ti,<EOL>state=ti.state))<EOL>ti_status.failed.add(key)<EOL>ti_status.to_run.pop(key)<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>return<EOL><DEDENT><DEDENT>backfill_context = DepContext(<EOL>deps=RUN_DEPS,<EOL>ignore_depends_on_past=ignore_depends_on_past,<EOL>ignore_task_deps=self.ignore_task_deps,<EOL>flag_upstream_failed=True)<EOL>if ti.are_dependencies_met(<EOL>dep_context=backfill_context,<EOL>session=session,<EOL>verbose=self.verbose):<EOL><INDENT>ti.refresh_from_db(lock_for_update=True, session=session)<EOL>if ti.state in (State.SCHEDULED, State.UP_FOR_RETRY, State.UP_FOR_RESCHEDULE):<EOL><INDENT>if executor.has_task(ti):<EOL><INDENT>self.log.debug(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>ti<EOL>)<EOL><DEDENT>else:<EOL><INDENT>self.log.debug('<STR_LIT>', ti)<EOL>ti.state = State.QUEUED<EOL>ti.queued_dttm = timezone.utcnow() if not ti.queued_dttm else ti.queued_dttm<EOL>session.merge(ti)<EOL>cfg_path = None<EOL>if executor.__class__ in (executors.LocalExecutor,<EOL>executors.SequentialExecutor):<EOL><INDENT>cfg_path = tmp_configuration_copy()<EOL><DEDENT>executor.queue_task_instance(<EOL>ti,<EOL>mark_success=self.mark_success,<EOL>pickle_id=pickle_id,<EOL>ignore_task_deps=self.ignore_task_deps,<EOL>ignore_depends_on_past=ignore_depends_on_past,<EOL>pool=self.pool,<EOL>cfg_path=cfg_path)<EOL>ti_status.running[key] = ti<EOL>ti_status.to_run.pop(key)<EOL><DEDENT><DEDENT>session.commit()<EOL>return<EOL><DEDENT>if ti.state == State.UPSTREAM_FAILED:<EOL><INDENT>self.log.error("<STR_LIT>", ti)<EOL>ti_status.failed.add(key)<EOL>ti_status.to_run.pop(key)<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>return<EOL><DEDENT>if ti.state == State.UP_FOR_RETRY:<EOL><INDENT>self.log.debug(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>", ti)<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>ti_status.to_run[key] = ti<EOL>return<EOL><DEDENT>if ti.state == State.UP_FOR_RESCHEDULE:<EOL><INDENT>self.log.debug(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>", ti)<EOL>if key in ti_status.running:<EOL><INDENT>ti_status.running.pop(key)<EOL><DEDENT>ti_status.to_run[key] = ti<EOL>return<EOL><DEDENT>self.log.debug('<STR_LIT>', ti)<EOL>ti_status.not_ready.add(key)<EOL><DEDENT>non_pool_slots = conf.getint('<STR_LIT>', '<STR_LIT>')<EOL>try:<EOL><INDENT>for task in self.dag.topological_sort():<EOL><INDENT>for key, ti in list(ti_status.to_run.items()):<EOL><INDENT>if task.task_id != ti.task_id:<EOL><INDENT>continue<EOL><DEDENT>if task.pool:<EOL><INDENT>pool = session.query(models.Pool).filter(models.Pool.pool == task.pool).first()<EOL>if not pool:<EOL><INDENT>raise PoolNotFound('<STR_LIT>'.format(task.pool))<EOL><DEDENT>open_slots = pool.open_slots(session=session)<EOL>if open_slots <= <NUM_LIT:0>:<EOL><INDENT>raise NoAvailablePoolSlot(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(<EOL>open_slots, task.pool))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if non_pool_slots <= <NUM_LIT:0>:<EOL><INDENT>raise NoAvailablePoolSlot(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>non_pool_slots -= <NUM_LIT:1><EOL><DEDENT>num_running_tasks = DAG.get_num_task_instances(<EOL>self.dag_id,<EOL>states=(State.QUEUED, State.RUNNING))<EOL>if num_running_tasks >= self.dag.concurrency:<EOL><INDENT>raise DagConcurrencyLimitReached(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>_per_task_process(task, key, ti)<EOL><DEDENT><DEDENT><DEDENT>except (NoAvailablePoolSlot, DagConcurrencyLimitReached) as e:<EOL><INDENT>self.log.debug(e)<EOL><DEDENT>self.heartbeat()<EOL>executor.heartbeat()<EOL>if (ti_status.not_ready and<EOL>ti_status.not_ready == set(ti_status.to_run) and<EOL>len(ti_status.running) == <NUM_LIT:0>):<EOL><INDENT>self.log.warning(<EOL>"<STR_LIT>",<EOL>ti_status.to_run.values()<EOL>)<EOL>ti_status.deadlocked.update(ti_status.to_run.values())<EOL>ti_status.to_run.clear()<EOL><DEDENT>self._manage_executor_state(ti_status.running)<EOL>self._update_counters(ti_status=ti_status)<EOL>_dag_runs = ti_status.active_runs[:]<EOL>for run in _dag_runs:<EOL><INDENT>run.update_state(session=session)<EOL>if run.state in State.finished():<EOL><INDENT>ti_status.finished_runs += <NUM_LIT:1><EOL>ti_status.active_runs.remove(run)<EOL>executed_run_dates.append(run.execution_date)<EOL><DEDENT><DEDENT>self._log_progress(ti_status)<EOL><DEDENT>return executed_run_dates<EOL>

Process a set of task instances from a set of dag runs. Special handling is done
to account for different task instance states that could be present when running
them in a backfill process.

:param ti_status: the internal status of the job
:type ti_status: BackfillJob._DagRunTaskStatus
:param executor: the executor to run the task instances
:type executor: BaseExecutor
:param pickle_id: the pickle_id if dag is pickled, None otherwise
:type pickle_id: int
:param start_date: the start date of the backfill job
:type start_date: datetime.datetime
:param session: the current session object
:type session: sqlalchemy.orm.session.Session
:return: the list of execution_dates for the finished dag runs
:rtype: list


@provide_session<EOL><INDENT>def _find_executable_task_instances(self, simple_dag_bag, states, session=None):<DEDENT>

executable_tis = []<EOL>TI = models.TaskInstance<EOL>DR = models.DagRun<EOL>DM = models.DagModel<EOL>ti_query = (<EOL>session<EOL>.query(TI)<EOL>.filter(TI.dag_id.in_(simple_dag_bag.dag_ids))<EOL>.outerjoin(<EOL>DR,<EOL>and_(DR.dag_id == TI.dag_id, DR.execution_date == TI.execution_date)<EOL>)<EOL>.filter(or_(DR.run_id == None,  <EOL>not_(DR.run_id.like(BackfillJob.ID_PREFIX + '<STR_LIT:%>'))))<EOL>.outerjoin(DM, DM.dag_id == TI.dag_id)<EOL>.filter(or_(DM.dag_id == None,  <EOL>not_(DM.is_paused)))<EOL>)<EOL>if None in states:<EOL><INDENT>ti_query = ti_query.filter(<EOL>or_(TI.state == None, TI.state.in_(states))  <EOL>)<EOL><DEDENT>else:<EOL><INDENT>ti_query = ti_query.filter(TI.state.in_(states))<EOL><DEDENT>task_instances_to_examine = ti_query.all()<EOL>if len(task_instances_to_examine) == <NUM_LIT:0>:<EOL><INDENT>self.log.debug("<STR_LIT>")<EOL>return executable_tis<EOL><DEDENT>task_instance_str = "<STR_LIT>".join(<EOL>[repr(x) for x in task_instances_to_examine])<EOL>self.log.info(<EOL>"<STR_LIT>", len(task_instances_to_examine),<EOL>task_instance_str<EOL>)<EOL>pools = {p.pool: p for p in session.query(models.Pool).all()}<EOL>pool_to_task_instances = defaultdict(list)<EOL>for task_instance in task_instances_to_examine:<EOL><INDENT>pool_to_task_instances[task_instance.pool].append(task_instance)<EOL><DEDENT>states_to_count_as_running = [State.RUNNING, State.QUEUED]<EOL>dag_concurrency_map, task_concurrency_map = self.__get_concurrency_maps(<EOL>states=states_to_count_as_running, session=session)<EOL>for pool, task_instances in pool_to_task_instances.items():<EOL><INDENT>pool_name = pool<EOL>if not pool:<EOL><INDENT>open_slots = models.Pool.default_pool_open_slots()<EOL>pool_name = models.Pool.default_pool_name<EOL><DEDENT>else:<EOL><INDENT>if pool not in pools:<EOL><INDENT>self.log.warning(<EOL>"<STR_LIT>",<EOL>pool<EOL>)<EOL>open_slots = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>open_slots = pools[pool].open_slots(session=session)<EOL><DEDENT><DEDENT>num_ready = len(task_instances)<EOL>self.log.info(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>pool, open_slots, num_ready<EOL>)<EOL>priority_sorted_task_instances = sorted(<EOL>task_instances, key=lambda ti: (-ti.priority_weight, ti.execution_date))<EOL>num_starving_tasks = <NUM_LIT:0><EOL>for current_index, task_instance in enumerate(priority_sorted_task_instances):<EOL><INDENT>if open_slots <= <NUM_LIT:0>:<EOL><INDENT>self.log.info(<EOL>"<STR_LIT>",<EOL>open_slots, pool<EOL>)<EOL>num_starving_tasks = len(priority_sorted_task_instances) - current_index<EOL>break<EOL><DEDENT>dag_id = task_instance.dag_id<EOL>simple_dag = simple_dag_bag.get_dag(dag_id)<EOL>current_dag_concurrency = dag_concurrency_map[dag_id]<EOL>dag_concurrency_limit = simple_dag_bag.get_dag(dag_id).concurrency<EOL>self.log.info(<EOL>"<STR_LIT>",<EOL>dag_id, current_dag_concurrency, dag_concurrency_limit<EOL>)<EOL>if current_dag_concurrency >= dag_concurrency_limit:<EOL><INDENT>self.log.info(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>task_instance, dag_id, dag_concurrency_limit<EOL>)<EOL>continue<EOL><DEDENT>task_concurrency_limit = simple_dag.get_task_special_arg(<EOL>task_instance.task_id,<EOL>'<STR_LIT>')<EOL>if task_concurrency_limit is not None:<EOL><INDENT>current_task_concurrency = task_concurrency_map[<EOL>(task_instance.dag_id, task_instance.task_id)<EOL>]<EOL>if current_task_concurrency >= task_concurrency_limit:<EOL><INDENT>self.log.info("<STR_LIT>"<EOL>"<STR_LIT>", task_instance)<EOL>continue<EOL><DEDENT><DEDENT>if self.executor.has_task(task_instance):<EOL><INDENT>self.log.debug(<EOL>"<STR_LIT>",<EOL>task_instance.key<EOL>)<EOL>continue<EOL><DEDENT>executable_tis.append(task_instance)<EOL>open_slots -= <NUM_LIT:1><EOL>dag_concurrency_map[dag_id] += <NUM_LIT:1><EOL>task_concurrency_map[(task_instance.dag_id, task_instance.task_id)] += <NUM_LIT:1><EOL><DEDENT>Stats.gauge('<STR_LIT>'.format(pool_name=pool_name),<EOL>num_starving_tasks)<EOL><DEDENT>task_instance_str = "<STR_LIT>".join(<EOL>[repr(x) for x in executable_tis])<EOL>self.log.info(<EOL>"<STR_LIT>", task_instance_str)<EOL>for ti in executable_tis:<EOL><INDENT>copy_dag_id = ti.dag_id<EOL>copy_execution_date = ti.execution_date<EOL>copy_task_id = ti.task_id<EOL>make_transient(ti)<EOL>ti.dag_id = copy_dag_id<EOL>ti.execution_date = copy_execution_date<EOL>ti.task_id = copy_task_id<EOL><DEDENT>return executable_tis<EOL>

Finds TIs that are ready for execution with respect to pool limits,
dag concurrency, executor state, and priority.

:param simple_dag_bag: TaskInstances associated with DAGs in the
    simple_dag_bag will be fetched from the DB and executed
:type simple_dag_bag: airflow.utils.dag_processing.SimpleDagBag
:param executor: the executor that runs task instances
:type executor: BaseExecutor
:param states: Execute TaskInstances in these states
:type states: tuple[airflow.utils.state.State]
:return: list[airflow.models.TaskInstance]


try:<EOL><INDENT>with create_session() as session:<EOL><INDENT>job = session.query(BaseJob).filter_by(id=self.id).one()<EOL>make_transient(job)<EOL>session.commit()<EOL><DEDENT>if job.state == State.SHUTDOWN:<EOL><INDENT>self.kill()<EOL><DEDENT>is_unit_test = conf.getboolean('<STR_LIT>', '<STR_LIT>')<EOL>if not is_unit_test:<EOL><INDENT>sleep_for = <NUM_LIT:0><EOL>if job.latest_heartbeat:<EOL><INDENT>seconds_remaining = self.heartrate -(timezone.utcnow() - job.latest_heartbeat).total_seconds()<EOL>sleep_for = max(<NUM_LIT:0>, seconds_remaining)<EOL><DEDENT>sleep(sleep_for)<EOL><DEDENT>with create_session() as session:<EOL><INDENT>job = session.query(BaseJob).filter(BaseJob.id == self.id).first()<EOL>job.latest_heartbeat = timezone.utcnow()<EOL>session.merge(job)<EOL>session.commit()<EOL>self.heartbeat_callback(session=session)<EOL>self.log.debug('<STR_LIT>')<EOL><DEDENT><DEDENT>except OperationalError as e:<EOL><INDENT>self.log.error("<STR_LIT>", str(e))<EOL><DEDENT>

Heartbeats update the job's entry in the database with a timestamp
for the latest_heartbeat and allows for the job to be killed
externally. This allows at the system level to monitor what is
actually active.

For instance, an old heartbeat for SchedulerJob would mean something
is wrong.

This also allows for any job to be killed externally, regardless
of who is running it or on which machine it is running.

Note that if your heartbeat is set to 60 seconds and you call this
method after 10 seconds of processing since the last heartbeat, it
will sleep 50 seconds to complete the 60 seconds and keep a steady
heart rate. If you go over 60 seconds before calling it, it won't
sleep at all.


if self._process is None:<EOL><INDENT>raise AirflowException("<STR_LIT>")<EOL><DEDENT>if self._done:<EOL><INDENT>return True<EOL><DEDENT>if self._result_queue and not self._result_queue.empty():<EOL><INDENT>self._result = self._result_queue.get_nowait()<EOL>self._done = True<EOL>self.log.debug("<STR_LIT>", self._process)<EOL>self._process.join()<EOL>return True<EOL><DEDENT>if self._result_queue and not self._process.is_alive():<EOL><INDENT>self._done = True<EOL>if not self._result_queue.empty():<EOL><INDENT>self._result = self._result_queue.get_nowait()<EOL><DEDENT>self.log.debug("<STR_LIT>", self._process)<EOL>self._process.join()<EOL>return True<EOL><DEDENT>return False<EOL>

Check if the process launched to process this file is done.

:return: whether the process is finished running
:rtype: bool


ti_status = BackfillJob._DagRunTaskStatus()<EOL>start_date = self.bf_start_date<EOL>run_dates = self.dag.get_run_dates(start_date=start_date,<EOL>end_date=self.bf_end_date)<EOL>if self.run_backwards:<EOL><INDENT>tasks_that_depend_on_past = [t.task_id for t in self.dag.task_dict.values() if t.depends_on_past]<EOL>if tasks_that_depend_on_past:<EOL><INDENT>raise AirflowException(<EOL>'<STR_LIT>'.format(<EOL>"<STR_LIT:U+002C>".join(tasks_that_depend_on_past)))<EOL><DEDENT>run_dates = run_dates[::-<NUM_LIT:1>]<EOL><DEDENT>if len(run_dates) == <NUM_LIT:0>:<EOL><INDENT>self.log.info("<STR_LIT>")<EOL>return<EOL><DEDENT>pickle_id = None<EOL>if not self.donot_pickle and self.executor.__class__ not in (<EOL>executors.LocalExecutor, executors.SequentialExecutor):<EOL><INDENT>pickle = DagPickle(self.dag)<EOL>session.add(pickle)<EOL>session.commit()<EOL>pickle_id = pickle.id<EOL><DEDENT>executor = self.executor<EOL>executor.start()<EOL>ti_status.total_runs = len(run_dates)  <EOL>try:<EOL><INDENT>remaining_dates = ti_status.total_runs<EOL>while remaining_dates > <NUM_LIT:0>:<EOL><INDENT>dates_to_process = [run_date for run_date in run_dates<EOL>if run_date not in ti_status.executed_dag_run_dates]<EOL>self._execute_for_run_dates(run_dates=dates_to_process,<EOL>ti_status=ti_status,<EOL>executor=executor,<EOL>pickle_id=pickle_id,<EOL>start_date=start_date,<EOL>session=session)<EOL>remaining_dates = (<EOL>ti_status.total_runs - len(ti_status.executed_dag_run_dates)<EOL>)<EOL>err = self._collect_errors(ti_status=ti_status, session=session)<EOL>if err:<EOL><INDENT>raise AirflowException(err)<EOL><DEDENT>if remaining_dates > <NUM_LIT:0>:<EOL><INDENT>self.log.info(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>self.dag_id<EOL>)<EOL>time.sleep(self.delay_on_limit_secs)<EOL><DEDENT><DEDENT><DEDENT>except (KeyboardInterrupt, SystemExit):<EOL><INDENT>self.log.warning("<STR_LIT>")<EOL>self._set_unfinished_dag_runs_to_failed(ti_status.active_runs)<EOL><DEDENT>finally:<EOL><INDENT>session.commit()<EOL>executor.end()<EOL><DEDENT>self.log.info("<STR_LIT>")<EOL>

Initializes all components required to run a dag for a specified date range and
calls helper method to execute the tasks.


def __init__(self, file_path, pickle_dags, dag_id_white_list, zombies):

self._file_path = file_path<EOL>self._result_queue = multiprocessing.Queue()<EOL>self._process = None<EOL>self._dag_id_white_list = dag_id_white_list<EOL>self._pickle_dags = pickle_dags<EOL>self._zombies = zombies<EOL>self._result = None<EOL>self._done = False<EOL>self._start_time = None<EOL>self._instance_id = DagFileProcessor.class_creation_counter<EOL>DagFileProcessor.class_creation_counter += <NUM_LIT:1><EOL>

:param file_path: a Python file containing Airflow DAG definitions
:type file_path: unicode
:param pickle_dags: whether to serialize the DAG objects to the DB
:type pickle_dags: bool
:param dag_id_whitelist: If specified, only look at these DAG ID's
:type dag_id_whitelist: list[unicode]
:param zombies: zombie task instances to kill
:type zombies: list[airflow.utils.dag_processing.SimpleTaskInstance]


<EOL>for dagbag_file in dagbag.file_last_changed:<EOL><INDENT>session.query(errors.ImportError).filter(<EOL>errors.ImportError.filename == dagbag_file<EOL>).delete()<EOL><DEDENT>for filename, stacktrace in six.iteritems(dagbag.import_errors):<EOL><INDENT>session.add(errors.ImportError(<EOL>filename=filename,<EOL>stacktrace=stacktrace))<EOL><DEDENT>session.commit()<EOL>

For the DAGs in the given DagBag, record any associated import errors and clears
errors for files that no longer have them. These are usually displayed through the
Airflow UI so that users know that there are issues parsing DAGs.

:param session: session for ORM operations
:type session: sqlalchemy.orm.session.Session
:param dagbag: DagBag containing DAGs with import errors
:type dagbag: airflow.models.DagBag


def send_lineage(self,<EOL>operator=None, inlets=None, outlets=None, context=None):

raise NotImplementedError()<EOL>

Sends lineage metadata to a backend
:param operator: the operator executing a transformation on the inlets and outlets
:param inlets: the inlets to this operator
:param outlets: the outlets from this operator
:param context: the current context of the task instance


return pendulum.parse(string, tz=timezone or TIMEZONE)<EOL>

Parse a time string and return an aware datetime
:param string: time string


if timezone is None:<EOL><INDENT>timezone = TIMEZONE<EOL><DEDENT>if is_naive(value):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>o = value.astimezone(timezone)<EOL>naive = dt.datetime(o.year,<EOL>o.month,<EOL>o.day,<EOL>o.hour,<EOL>o.minute,<EOL>o.second,<EOL>o.microsecond)<EOL>return naive<EOL>

Make an aware datetime.datetime naive in a given time zone.

:param value: datetime
:param timezone: timezone
:return: naive datetime


if '<STR_LIT>' not in kwargs:<EOL><INDENT>kwargs['<STR_LIT>'] = TIMEZONE<EOL><DEDENT>return dt.datetime(*args, **kwargs)<EOL>

Wrapper around datetime.datetime that adds settings.TIMEZONE if tzinfo not specified

:return: datetime.datetime


<EOL>d = dt.datetime(<NUM_LIT>, <NUM_LIT:1>, <NUM_LIT:1>)<EOL>d = d.replace(tzinfo=utc)<EOL>return d<EOL>

Gets the epoch in the users timezone
:return:


params = dict()<EOL>if in_env_var_format:<EOL><INDENT>name_format = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>name_format = '<STR_LIT:default>'<EOL><DEDENT>task_instance = context.get('<STR_LIT>')<EOL>if task_instance and task_instance.dag_id:<EOL><INDENT>params[AIRFLOW_VAR_NAME_FORMAT_MAPPING['<STR_LIT>'][<EOL>name_format]] = task_instance.dag_id<EOL><DEDENT>if task_instance and task_instance.task_id:<EOL><INDENT>params[AIRFLOW_VAR_NAME_FORMAT_MAPPING['<STR_LIT>'][<EOL>name_format]] = task_instance.task_id<EOL><DEDENT>if task_instance and task_instance.execution_date:<EOL><INDENT>params[<EOL>AIRFLOW_VAR_NAME_FORMAT_MAPPING['<STR_LIT>'][<EOL>name_format]] = task_instance.execution_date.isoformat()<EOL><DEDENT>dag_run = context.get('<STR_LIT>')<EOL>if dag_run and dag_run.run_id:<EOL><INDENT>params[AIRFLOW_VAR_NAME_FORMAT_MAPPING['<STR_LIT>'][<EOL>name_format]] = dag_run.run_id<EOL><DEDENT>return params<EOL>

Given a context, this function provides a dictionary of values that can be used to
externally reconstruct relations between dags, dag_runs, tasks and task_instances.
Default to abc.def.ghi format and can be made to ABC_DEF_GHI format if
in_env_var_format is set to True.

:param context: The context for the task_instance of interest.
:type context: dict
:param in_env_var_format: If returned vars should be in ABC_DEF_GHI format.
:type in_env_var_format: bool
:return: task_instance context as dict.


<EOL>if self.closed:<EOL><INDENT>return<EOL><DEDENT>super().close()<EOL>if not self.upload_on_close:<EOL><INDENT>return<EOL><DEDENT>local_loc = os.path.join(self.local_base, self.log_relative_path)<EOL>remote_loc = os.path.join(self.remote_base, self.log_relative_path)<EOL>if os.path.exists(local_loc):<EOL><INDENT>with open(local_loc, '<STR_LIT:r>') as logfile:<EOL><INDENT>log = logfile.read()<EOL><DEDENT>self.s3_write(log, remote_loc)<EOL><DEDENT>self.closed = True<EOL>

Close and upload local log file to remote storage S3.


def _read(self, ti, try_number, metadata=None):

<EOL>log_relative_path = self._render_filename(ti, try_number)<EOL>remote_loc = os.path.join(self.remote_base, log_relative_path)<EOL>if self.s3_log_exists(remote_loc):<EOL><INDENT>remote_log = self.s3_read(remote_loc, return_error=True)<EOL>log = '<STR_LIT>'.format(<EOL>remote_loc, remote_log)<EOL>return log, {'<STR_LIT>': True}<EOL><DEDENT>else:<EOL><INDENT>return super()._read(ti, try_number)<EOL><DEDENT>

Read logs of given task instance and try_number from S3 remote storage.
If failed, read the log from task instance host machine.
:param ti: task instance object
:param try_number: task instance try_number to read logs from
:param metadata: log metadata,
                 can be used for steaming log reading and auto-tailing.


<EOL>log_relative_path = self._render_filename(ti, try_number)<EOL>remote_loc = os.path.join(self.remote_base, log_relative_path)<EOL>if self.wasb_log_exists(remote_loc):<EOL><INDENT>remote_log = self.wasb_read(remote_loc, return_error=True)<EOL>log = '<STR_LIT>'.format(<EOL>remote_loc, remote_log)<EOL>return log, {'<STR_LIT>': True}<EOL><DEDENT>else:<EOL><INDENT>return super()._read(ti, try_number)<EOL><DEDENT>

Read logs of given task instance and try_number from Wasb remote storage.
If failed, read the log from task instance host machine.
:param ti: task instance object
:param try_number: task instance try_number to read logs from
:param metadata: log metadata,
                 can be used for steaming log reading and auto-tailing.


try:<EOL><INDENT>return self.hook.check_for_blob(self.wasb_container, remote_log_location)<EOL><DEDENT>except Exception:<EOL><INDENT>pass<EOL><DEDENT>return False<EOL>

Check if remote_log_location exists in remote storage
:param remote_log_location: log's location in remote storage
:return: True if location exists else False


def wasb_write(self, log, remote_log_location, append=True):

if append and self.wasb_log_exists(remote_log_location):<EOL><INDENT>old_log = self.wasb_read(remote_log_location)<EOL>log = '<STR_LIT:\n>'.join([old_log, log]) if old_log else log<EOL><DEDENT>try:<EOL><INDENT>self.hook.load_string(<EOL>log,<EOL>self.wasb_container,<EOL>remote_log_location,<EOL>)<EOL><DEDENT>except AzureHttpError:<EOL><INDENT>self.log.exception('<STR_LIT>',<EOL>remote_log_location)<EOL><DEDENT>

Writes the log to the remote_log_location. Fails silently if no hook
was created.
:param log: the log to write to the remote_log_location
:type log: str
:param remote_log_location: the log's location in remote storage
:type remote_log_location: str (path)
:param append: if False, any existing log file is overwritten. If True,
    the new log is appended to any existing logs.
:type append: bool


if len(self._buffer) > <NUM_LIT:0>:<EOL><INDENT>self.logger.log(self.level, self._buffer)<EOL>self._buffer = str()<EOL><DEDENT>

Ensure all logging output has been flushed


return False<EOL>

Returns False to indicate that the stream is not closed (as it will be
open for the duration of Airflow's lifecycle).

For compatibility with the io.IOBase interface.


<EOL>log_relative_path = self._render_filename(ti, try_number)<EOL>remote_loc = os.path.join(self.remote_base, log_relative_path)<EOL>try:<EOL><INDENT>remote_log = self.gcs_read(remote_loc)<EOL>log = '<STR_LIT>'.format(<EOL>remote_loc, remote_log)<EOL>return log, {'<STR_LIT>': True}<EOL><DEDENT>except Exception as e:<EOL><INDENT>log = '<STR_LIT>'.format(<EOL>remote_loc, str(e))<EOL>self.log.error(log)<EOL>local_log, metadata = super()._read(ti, try_number)<EOL>log += local_log<EOL>return log, metadata<EOL><DEDENT>

Read logs of given task instance and try_number from GCS.
If failed, read the log from task instance host machine.
:param ti: task instance object
:param try_number: task instance try_number to read logs from
:param metadata: log metadata,
                 can be used for steaming log reading and auto-tailing.


metrics = {'<STR_LIT>': func_name, '<STR_LIT>': datetime.utcnow(),<EOL>'<STR_LIT>': '<STR_LIT:{}>'.format(list(sys.argv)), '<STR_LIT:user>': getpass.getuser()}<EOL>assert isinstance(namespace, Namespace)<EOL>tmp_dic = vars(namespace)<EOL>metrics['<STR_LIT>'] = tmp_dic.get('<STR_LIT>')<EOL>metrics['<STR_LIT>'] = tmp_dic.get('<STR_LIT>')<EOL>metrics['<STR_LIT>'] = tmp_dic.get('<STR_LIT>')<EOL>metrics['<STR_LIT>'] = socket.gethostname()<EOL>extra = json.dumps(dict((k, metrics[k]) for k in ('<STR_LIT>', '<STR_LIT>')))<EOL>log = Log(<EOL>event='<STR_LIT>'.format(func_name),<EOL>task_instance=None,<EOL>owner=metrics['<STR_LIT:user>'],<EOL>extra=extra,<EOL>task_id=metrics.get('<STR_LIT>'),<EOL>dag_id=metrics.get('<STR_LIT>'),<EOL>execution_date=metrics.get('<STR_LIT>'))<EOL>metrics['<STR_LIT>'] = log<EOL>return metrics<EOL>

Builds metrics dict from function args
It assumes that function arguments is from airflow.bin.cli module's function
and has Namespace instance where it optionally contains "dag_id", "task_id",
and "execution_date".

:param func_name: name of function
:param namespace: Namespace instance from argparse
:return: dict with metrics


@functools.wraps(f)<EOL>def wrapper(*args, **kwargs):<EOL><INDENT>"""<STR_LIT>"""<EOL>assert args<EOL>assert isinstance(args[<NUM_LIT:0>], Namespace),"<STR_LIT>""<STR_LIT>".format(args[<NUM_LIT:0>])<EOL>metrics = _build_metrics(f.__name__, args[<NUM_LIT:0>])<EOL>cli_action_loggers.on_pre_execution(**metrics)<EOL>try:<EOL><INDENT>return f(*args, **kwargs)<EOL><DEDENT>except Exception as e:<EOL><INDENT>metrics['<STR_LIT:error>'] = e<EOL>raise<EOL><DEDENT>finally:<EOL><INDENT>metrics['<STR_LIT>'] = datetime.utcnow()<EOL>cli_action_loggers.on_post_execution(**metrics)<EOL><DEDENT><DEDENT>return wrapper<EOL>

Decorates function to execute function at the same time submitting action_logging
but in CLI context. It will call action logger callbacks twice,
one for pre-execution and the other one for post-execution.

Action logger will be called with below keyword parameters:
    sub_command : name of sub-command
    start_datetime : start datetime instance by utc
    end_datetime : end datetime instance by utc
    full_command : full command line arguments
    user : current user
    log : airflow.models.log.Log ORM instance
    dag_id : dag id (optional)
    task_id : task_id (optional)
    execution_date : execution date (optional)
    error : exception instance if there's an exception

:param f: function instance
:return: wrapped function


def days_ago(n, hour=<NUM_LIT:0>, minute=<NUM_LIT:0>, second=<NUM_LIT:0>, microsecond=<NUM_LIT:0>):

today = timezone.utcnow().replace(<EOL>hour=hour,<EOL>minute=minute,<EOL>second=second,<EOL>microsecond=microsecond)<EOL>return today - timedelta(days=n)<EOL>

Get a datetime object representing `n` days ago. By default the time is
set to midnight.


def date_range(start_date, end_date=None, num=None, delta=None):

if not delta:<EOL><INDENT>return []<EOL><DEDENT>if end_date and start_date > end_date:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if end_date and num:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if not end_date and not num:<EOL><INDENT>end_date = timezone.utcnow()<EOL><DEDENT>delta_iscron = False<EOL>tz = start_date.tzinfo<EOL>if isinstance(delta, six.string_types):<EOL><INDENT>delta_iscron = True<EOL>start_date = timezone.make_naive(start_date, tz)<EOL>cron = croniter(delta, start_date)<EOL><DEDENT>elif isinstance(delta, timedelta):<EOL><INDENT>delta = abs(delta)<EOL><DEDENT>dates = []<EOL>if end_date:<EOL><INDENT>if timezone.is_naive(start_date):<EOL><INDENT>end_date = timezone.make_naive(end_date, tz)<EOL><DEDENT>while start_date <= end_date:<EOL><INDENT>if timezone.is_naive(start_date):<EOL><INDENT>dates.append(timezone.make_aware(start_date, tz))<EOL><DEDENT>else:<EOL><INDENT>dates.append(start_date)<EOL><DEDENT>if delta_iscron:<EOL><INDENT>start_date = cron.get_next(datetime)<EOL><DEDENT>else:<EOL><INDENT>start_date += delta<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for _ in range(abs(num)):<EOL><INDENT>if timezone.is_naive(start_date):<EOL><INDENT>dates.append(timezone.make_aware(start_date, tz))<EOL><DEDENT>else:<EOL><INDENT>dates.append(start_date)<EOL><DEDENT>if delta_iscron:<EOL><INDENT>if num > <NUM_LIT:0>:<EOL><INDENT>start_date = cron.get_next(datetime)<EOL><DEDENT>else:<EOL><INDENT>start_date = cron.get_prev(datetime)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if num > <NUM_LIT:0>:<EOL><INDENT>start_date += delta<EOL><DEDENT>else:<EOL><INDENT>start_date -= delta<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return sorted(dates)<EOL>

Get a set of dates as a list based on a start, end and delta, delta
can be something that can be added to `datetime.datetime`
or a cron expression as a `str`

:Example::

    date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta=timedelta(1))
        [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0),
        datetime.datetime(2016, 1, 3, 0, 0)]
    date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta='0 0 * * *')
        [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0),
        datetime.datetime(2016, 1, 3, 0, 0)]
    date_range(datetime(2016, 1, 1), datetime(2016, 3, 3), delta="0 0 0 * *")
        [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 2, 1, 0, 0),
        datetime.datetime(2016, 3, 1, 0, 0)]

:param start_date: anchor date to start the series from
:type start_date: datetime.datetime
:param end_date: right boundary for the date range
:type end_date: datetime.datetime
:param num: alternatively to end_date, you can specify the number of
    number of entries you want in the range. This number can be negative,
    output will always be sorted regardless
:type num: int


def send_email(to, subject, html_content,<EOL>files=None, dryrun=False, cc=None, bcc=None,<EOL>mime_subtype='<STR_LIT>', mime_charset='<STR_LIT:utf-8>', **kwargs):

path, attr = configuration.conf.get('<STR_LIT:email>', '<STR_LIT>').rsplit('<STR_LIT:.>', <NUM_LIT:1>)<EOL>module = importlib.import_module(path)<EOL>backend = getattr(module, attr)<EOL>to = get_email_address_list(to)<EOL>to = "<STR_LIT:U+002CU+0020>".join(to)<EOL>return backend(to, subject, html_content, files=files,<EOL>dryrun=dryrun, cc=cc, bcc=bcc,<EOL>mime_subtype=mime_subtype, mime_charset=mime_charset, **kwargs)<EOL>

Send email using backend specified in EMAIL_BACKEND.


cfg_dict = conf.as_dict(display_sensitive=True, raw=True)<EOL>temp_fd, cfg_path = mkstemp()<EOL>with os.fdopen(temp_fd, '<STR_LIT:w>') as temp_file:<EOL><INDENT>if chmod is not None:<EOL><INDENT>os.fchmod(temp_fd, chmod)<EOL><DEDENT>json.dump(cfg_dict, temp_file)<EOL><DEDENT>return cfg_path<EOL>

Returns a path for a temporary file including a full copy of the configuration
settings.
:return: a path to a temporary file


for up_task, down_task in zip(tasks[:-<NUM_LIT:1>], tasks[<NUM_LIT:1>:]):<EOL><INDENT>up_task.set_downstream(down_task)<EOL><DEDENT>

Given a number of tasks, builds a dependency chain.

chain(task_1, task_2, task_3, task_4)

is equivalent to

task_1.set_downstream(task_2)
task_2.set_downstream(task_3)
task_3.set_downstream(task_4)


if not rows:<EOL><INDENT>return<EOL><DEDENT>if hasattr(rows[<NUM_LIT:0>], '<STR_LIT>'):  <EOL><INDENT>headers = rows[<NUM_LIT:0>]._fields<EOL><DEDENT>else:<EOL><INDENT>headers = ["<STR_LIT>".format(i) for i in range(len(rows[<NUM_LIT:0>]))]<EOL><DEDENT>lens = [len(s) for s in headers]<EOL>for row in rows:<EOL><INDENT>for i in range(len(rows[<NUM_LIT:0>])):<EOL><INDENT>slenght = len("<STR_LIT:{}>".format(row[i]))<EOL>if slenght > lens[i]:<EOL><INDENT>lens[i] = slenght<EOL><DEDENT><DEDENT><DEDENT>formats = []<EOL>hformats = []<EOL>for i in range(len(rows[<NUM_LIT:0>])):<EOL><INDENT>if isinstance(rows[<NUM_LIT:0>][i], int):<EOL><INDENT>formats.append("<STR_LIT>" % lens[i])<EOL><DEDENT>else:<EOL><INDENT>formats.append("<STR_LIT>" % lens[i])<EOL><DEDENT>hformats.append("<STR_LIT>" % lens[i])<EOL><DEDENT>pattern = "<STR_LIT>".join(formats)<EOL>hpattern = "<STR_LIT>".join(hformats)<EOL>separator = "<STR_LIT>".join(['<STR_LIT:->' * n for n in lens])<EOL>s = "<STR_LIT>"<EOL>s += separator + '<STR_LIT:\n>'<EOL>s += (hpattern % tuple(headers)) + '<STR_LIT:\n>'<EOL>s += separator + '<STR_LIT:\n>'<EOL>def f(t):<EOL><INDENT>return "<STR_LIT:{}>".format(t) if isinstance(t, basestring) else t<EOL><DEDENT>for line in rows:<EOL><INDENT>s += pattern % tuple(f(t) for t in line) + '<STR_LIT:\n>'<EOL><DEDENT>s += separator + '<STR_LIT:\n>'<EOL>return s<EOL>

Returns a pretty ascii table from tuples

    If namedtuple are used, the table will have headers


return self._concurrency<EOL>

:return: maximum number of tasks that can run simultaneously from this DAG
:rtype: int


return [x.pid for x in self._processors.values()]<EOL>

:return: a list of the PIDs for the processors that are running
:rtype: List[int]


return self.dag_id_to_simple_dag.keys()<EOL>

:return: IDs of all the DAGs in this
:rtype: list[unicode]


elapsed_time_since_refresh = (timezone.utcnow() -<EOL>self.last_dag_dir_refresh_time).total_seconds()<EOL>if elapsed_time_since_refresh > self.dag_dir_list_interval:<EOL><INDENT>self.log.info("<STR_LIT>", self._dag_directory)<EOL>self._file_paths = list_py_file_paths(self._dag_directory)<EOL>self.last_dag_dir_refresh_time = timezone.utcnow()<EOL>self.log.info("<STR_LIT>", len(self._file_paths), self._dag_directory)<EOL>self.set_file_paths(self._file_paths)<EOL>try:<EOL><INDENT>self.log.debug("<STR_LIT>")<EOL>self.clear_nonexistent_import_errors()<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.exception("<STR_LIT>")<EOL><DEDENT><DEDENT>

Refresh file paths from dag dir if we haven't done it for too long.


return len(self._processors)<EOL>

:return: the number of files currently being processed
:rtype: int


raise NotImplementedError()<EOL>

:return: the path to the file that this is processing
:rtype: unicode


<EOL>self._sync_metadata()<EOL>self._heartbeat_manager()<EOL>simple_dags = []<EOL>if sys.platform == "<STR_LIT>":<EOL><INDENT>qsize = self._result_count<EOL><DEDENT>else:<EOL><INDENT>qsize = self._result_queue.qsize()<EOL><DEDENT>for _ in range(qsize):<EOL><INDENT>simple_dags.append(self._result_queue.get())<EOL><DEDENT>self._result_count = <NUM_LIT:0><EOL>return simple_dags<EOL>

Harvest DAG parsing results from result queue and sync metadata from stat queue.
:return: List of parsing result in SimpleDag format.


raise NotImplementedError()<EOL>

Terminate (and then kill) the process launched to process the file


query = session.query(errors.ImportError)<EOL>if self._file_paths:<EOL><INDENT>query = query.filter(<EOL>~errors.ImportError.filename.in_(self._file_paths)<EOL>)<EOL><DEDENT>query.delete(synchronize_session='<STR_LIT>')<EOL>session.commit()<EOL>

Clears import errors for files that no longer exist.

:param session: session for ORM operations
:type session: sqlalchemy.orm.session.Session


self._file_paths = file_paths<EOL>self._file_path_queue = []<EOL>self._dag_directory = dag_directory<EOL>self._max_runs = max_runs<EOL>self._processor_factory = processor_factory<EOL>self._signal_conn = signal_conn<EOL>self._stat_queue = stat_queue<EOL>self._result_queue = result_queue<EOL>self._async_mode = async_mode<EOL>self._parallelism = conf.getint('<STR_LIT>', '<STR_LIT>')<EOL>if '<STR_LIT>' in conf.get('<STR_LIT>', '<STR_LIT>') and self._parallelism > <NUM_LIT:1>:<EOL><INDENT>self.log.error("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>self._parallelism = <NUM_LIT:1><EOL><DEDENT>self._file_process_interval = conf.getint('<STR_LIT>',<EOL>'<STR_LIT>')<EOL>self.print_stats_interval = conf.getint('<STR_LIT>',<EOL>'<STR_LIT>')<EOL>self._zombie_threshold_secs = (<EOL>conf.getint('<STR_LIT>', '<STR_LIT>'))<EOL>self._processors = {}<EOL>self._last_runtime = {}<EOL>self._last_finish_time = {}<EOL>self._last_zombie_query_time = timezone.utcnow()<EOL>self.last_dag_dir_refresh_time = timezone.utcnow()<EOL>self.last_stat_print_time = timezone.datetime(<NUM_LIT>, <NUM_LIT:1>, <NUM_LIT:1>)<EOL>self._zombie_query_interval = <NUM_LIT:10><EOL>self._run_count = defaultdict(int)<EOL>self._heart_beat_key = '<STR_LIT>'<EOL>self.dag_dir_list_interval = conf.getint('<STR_LIT>',<EOL>'<STR_LIT>')<EOL>self._log = logging.getLogger('<STR_LIT>')<EOL>signal.signal(signal.SIGINT, self._exit_gracefully)<EOL>signal.signal(signal.SIGTERM, self._exit_gracefully)<EOL>

:param dag_directory: Directory where DAG definitions are kept. All
    files in file_paths should be under this directory
:type dag_directory: unicode
:param file_paths: list of file paths that contain DAG definitions
:type file_paths: list[unicode]
:param max_runs: The number of times to parse and schedule each file. -1
    for unlimited.
:type max_runs: int
:param processor_factory: function that creates processors for DAG
    definition files. Arguments are (dag_definition_path)
:type processor_factory: (unicode, unicode, list) -> (AbstractDagFileProcessor)
:param signal_conn: connection to communicate signal with processor agent.
:type signal_conn: airflow.models.connection.Connection
:param stat_queue: the queue to use for passing back parsing stat to agent.
:type stat_queue: multiprocessing.Queue
:param result_queue: the queue to use for passing back the result to agent.
:type result_queue: multiprocessing.Queue
:param async_mode: whether to start the manager in async mode
:type async_mode: bool


return self._last_finish_time.get(file_path)<EOL>

:param file_path: the path to the file that was processed
:type file_path: unicode
:return: the finish time of the process of the last run, or None if the
    file was never processed.
:rtype: datetime


_, archive, filename = re.search(<EOL>r'<STR_LIT>'.format(re.escape(os.sep)), fileloc).groups()<EOL>if archive and zipfile.is_zipfile(archive):<EOL><INDENT>return archive<EOL><DEDENT>else:<EOL><INDENT>return fileloc<EOL><DEDENT>

If the path contains a folder with a .zip suffix, then
the folder is treated as a zip archive and path to zip is returned.


for processor in self._processors.values():<EOL><INDENT>processor.terminate()<EOL><DEDENT>

Stops all running processors
:return: None


with create_session() as session:<EOL><INDENT>session.add(log)<EOL><DEDENT>

A default action logger callback that behave same as www.utils.action_logging
which uses global session and pushes log ORM object.
:param log: An log ORM instance
:param **_: other keyword arguments that is not being used by this function
:return: None


logging.debug("<STR_LIT>", action_logger)<EOL>__post_exec_callbacks.append(action_logger)<EOL>

Registers more action_logger function callback for post-execution.
This function callback is expected to be called with keyword args.
For more about the arguments that is being passed to the callback,
refer to airflow.utils.cli.action_logging()
:param action_logger: An action logger function
:return: None


def get_first(self, hql, parameters=None):

try:<EOL><INDENT>return super().get_first(<EOL>self._strip_sql(hql), parameters)<EOL><DEDENT>except DatabaseError as e:<EOL><INDENT>raise PrestoException(self._get_pretty_exception_message(e))<EOL><DEDENT>

Returns only the first row, regardless of how many rows the query
returns.


return super().run(self._strip_sql(hql), parameters)<EOL>

Execute the statement against Presto. Can be used to create views.


def set_autocommit(self, conn, autocommit):

if not self.supports_autocommit and autocommit:<EOL><INDENT>self.log.warn(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>"),<EOL>getattr(self, self.conn_name_attr))<EOL><DEDENT>conn.autocommit = autocommit<EOL>

Sets the autocommit flag on the connection


def run_and_check(self, session, prepped_request, extra_options):

extra_options = extra_options or {}<EOL>try:<EOL><INDENT>response = session.send(<EOL>prepped_request,<EOL>stream=extra_options.get("<STR_LIT>", False),<EOL>verify=extra_options.get("<STR_LIT>", True),<EOL>proxies=extra_options.get("<STR_LIT>", {}),<EOL>cert=extra_options.get("<STR_LIT>"),<EOL>timeout=extra_options.get("<STR_LIT>"),<EOL>allow_redirects=extra_options.get("<STR_LIT>", True))<EOL>if extra_options.get('<STR_LIT>', True):<EOL><INDENT>self.check_response(response)<EOL><DEDENT>return response<EOL><DEDENT>except requests.exceptions.ConnectionError as ex:<EOL><INDENT>self.log.warn(str(ex) + '<STR_LIT>')<EOL>raise ex<EOL><DEDENT>

Grabs extra options like timeout and actually runs the request,
checking for the result

:param session: the session to be used to execute the request
:type session: requests.Session
:param prepped_request: the prepared request generated in run()
:type prepped_request: session.prepare_request
:param extra_options: additional options to be used when executing the request
    i.e. {'check_response': False} to avoid checking raising exceptions on non 2XX
    or 3XX status codes
:type extra_options: dict


def insert_rows(self, table, rows, target_fields=None, commit_every=<NUM_LIT:1000>):

if target_fields:<EOL><INDENT>target_fields = '<STR_LIT:U+002CU+0020>'.join(target_fields)<EOL>target_fields = '<STR_LIT>'.format(target_fields)<EOL><DEDENT>else:<EOL><INDENT>target_fields = '<STR_LIT>'<EOL><DEDENT>conn = self.get_conn()<EOL>cur = conn.cursor()<EOL>if self.supports_autocommit:<EOL><INDENT>cur.execute('<STR_LIT>')<EOL><DEDENT>conn.commit()<EOL>i = <NUM_LIT:0><EOL>for row in rows:<EOL><INDENT>i += <NUM_LIT:1><EOL>lst = []<EOL>for cell in row:<EOL><INDENT>if isinstance(cell, basestring):<EOL><INDENT>lst.append("<STR_LIT:'>" + str(cell).replace("<STR_LIT:'>", "<STR_LIT>") + "<STR_LIT:'>")<EOL><DEDENT>elif cell is None:<EOL><INDENT>lst.append('<STR_LIT>')<EOL><DEDENT>elif type(cell) == float andnumpy.isnan(cell):  <EOL><INDENT>lst.append('<STR_LIT>')<EOL><DEDENT>elif isinstance(cell, numpy.datetime64):<EOL><INDENT>lst.append("<STR_LIT:'>" + str(cell) + "<STR_LIT:'>")<EOL><DEDENT>elif isinstance(cell, datetime):<EOL><INDENT>lst.append("<STR_LIT>" +<EOL>cell.strftime('<STR_LIT>') +<EOL>"<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>lst.append(str(cell))<EOL><DEDENT><DEDENT>values = tuple(lst)<EOL>sql = '<STR_LIT>''<STR_LIT>'.format(table,<EOL>target_fields,<EOL>'<STR_LIT:U+002C>'.join(values))<EOL>cur.execute(sql)<EOL>if i % commit_every == <NUM_LIT:0>:<EOL><INDENT>conn.commit()<EOL>self.log.info('<STR_LIT>', i, table)<EOL><DEDENT><DEDENT>conn.commit()<EOL>cur.close()<EOL>conn.close()<EOL>self.log.info('<STR_LIT>', i)<EOL>

A generic way to insert a set of tuples into a table,
the whole set of inserts is treated as one transaction
Changes from standard DbApiHook implementation:

- Oracle SQL queries in cx_Oracle can not be terminated with a semicolon (`;`)
- Replace NaN values with NULL using `numpy.nan_to_num` (not using
  `is_nan()` because of input types error for strings)
- Coerce datetime cells to Oracle DATETIME format during insert

:param table: target Oracle table, use dot notation to target a
    specific database
:type table: str
:param rows: the rows to insert into the table
:type rows: iterable of tuples
:param target_fields: the names of the columns to fill in the table
:type target_fields: iterable of str
:param commit_every: the maximum number of rows to insert in one transaction
    Default 1000, Set greater than 0.
    Set 1 to insert each row in each single transaction
:type commit_every: int


if input_serialization is None:<EOL><INDENT>input_serialization = {'<STR_LIT>': {}}<EOL><DEDENT>if output_serialization is None:<EOL><INDENT>output_serialization = {'<STR_LIT>': {}}<EOL><DEDENT>if not bucket_name:<EOL><INDENT>(bucket_name, key) = self.parse_s3_url(key)<EOL><DEDENT>response = self.get_conn().select_object_content(<EOL>Bucket=bucket_name,<EOL>Key=key,<EOL>Expression=expression,<EOL>ExpressionType=expression_type,<EOL>InputSerialization=input_serialization,<EOL>OutputSerialization=output_serialization)<EOL>return '<STR_LIT>'.join(event['<STR_LIT>']['<STR_LIT>'].decode('<STR_LIT:utf-8>')<EOL>for event in response['<STR_LIT>']<EOL>if '<STR_LIT>' in event)<EOL>

Reads a key with S3 Select.

:param key: S3 key that will point to the file
:type key: str
:param bucket_name: Name of the bucket in which the file is stored
:type bucket_name: str
:param expression: S3 Select expression
:type expression: str
:param expression_type: S3 Select expression type
:type expression_type: str
:param input_serialization: S3 Select input data serialization format
:type input_serialization: dict
:param output_serialization: S3 Select output data serialization format
:type output_serialization: dict
:return: retrieved subset of original data by S3 Select
:rtype: str

.. seealso::
    For more details about S3 Select parameters:
    http://boto3.readthedocs.io/en/latest/reference/services/s3.html#S3.Client.select_object_content


def check_for_prefix(self, bucket_name, prefix, delimiter):

prefix = prefix + delimiter if prefix[-<NUM_LIT:1>] != delimiter else prefix<EOL>prefix_split = re.split(r'<STR_LIT>'.format(d=delimiter), prefix, <NUM_LIT:1>)<EOL>previous_level = prefix_split[<NUM_LIT:0>]<EOL>plist = self.list_prefixes(bucket_name, previous_level, delimiter)<EOL>return False if plist is None else prefix in plist<EOL>

Checks that a prefix exists in a bucket

:param bucket_name: the name of the bucket
:type bucket_name: str
:param prefix: a key prefix
:type prefix: str
:param delimiter: the delimiter marks key hierarchy.
:type delimiter: str


conn = self.get_connection(self.mysql_conn_id)<EOL>conn_config = {<EOL>"<STR_LIT:user>": conn.login,<EOL>"<STR_LIT>": conn.password or '<STR_LIT>',<EOL>"<STR_LIT:host>": conn.host or '<STR_LIT:localhost>',<EOL>"<STR_LIT>": self.schema or conn.schema or '<STR_LIT>'<EOL>}<EOL>if not conn.port:<EOL><INDENT>conn_config["<STR_LIT:port>"] = <NUM_LIT><EOL><DEDENT>else:<EOL><INDENT>conn_config["<STR_LIT:port>"] = int(conn.port)<EOL><DEDENT>if conn.extra_dejson.get('<STR_LIT>', False):<EOL><INDENT>conn_config["<STR_LIT>"] = conn.extra_dejson["<STR_LIT>"]<EOL>if (conn_config["<STR_LIT>"]).lower() == '<STR_LIT:utf8>' or(conn_config["<STR_LIT>"]).lower() == '<STR_LIT:utf-8>':<EOL><INDENT>conn_config["<STR_LIT>"] = True<EOL><DEDENT><DEDENT>if conn.extra_dejson.get('<STR_LIT>', False):<EOL><INDENT>if (conn.extra_dejson["<STR_LIT>"]).lower() == '<STR_LIT>':<EOL><INDENT>conn_config["<STR_LIT>"] = MySQLdb.cursors.SSCursor<EOL><DEDENT>elif (conn.extra_dejson["<STR_LIT>"]).lower() == '<STR_LIT>':<EOL><INDENT>conn_config["<STR_LIT>"] = MySQLdb.cursors.DictCursor<EOL><DEDENT>elif (conn.extra_dejson["<STR_LIT>"]).lower() == '<STR_LIT>':<EOL><INDENT>conn_config["<STR_LIT>"] = MySQLdb.cursors.SSDictCursor<EOL><DEDENT><DEDENT>local_infile = conn.extra_dejson.get('<STR_LIT>', False)<EOL>if conn.extra_dejson.get('<STR_LIT>', False):<EOL><INDENT>dejson_ssl = conn.extra_dejson['<STR_LIT>']<EOL>if isinstance(dejson_ssl, six.string_types):<EOL><INDENT>dejson_ssl = json.loads(dejson_ssl)<EOL><DEDENT>conn_config['<STR_LIT>'] = dejson_ssl<EOL><DEDENT>if conn.extra_dejson.get('<STR_LIT>'):<EOL><INDENT>conn_config['<STR_LIT>'] = conn.extra_dejson['<STR_LIT>']<EOL><DEDENT>if local_infile:<EOL><INDENT>conn_config["<STR_LIT>"] = <NUM_LIT:1><EOL><DEDENT>conn = MySQLdb.connect(**conn_config)<EOL>return conn<EOL>


with TemporaryDirectory(prefix='<STR_LIT>') as tmp_dir:<EOL><INDENT>with NamedTemporaryFile(dir=tmp_dir) as f:<EOL><INDENT>f.write(pig.encode('<STR_LIT:utf-8>'))<EOL>f.flush()<EOL>fname = f.name<EOL>pig_bin = '<STR_LIT>'<EOL>cmd_extra = []<EOL>pig_cmd = [pig_bin, '<STR_LIT>', fname] + cmd_extra<EOL>if self.pig_properties:<EOL><INDENT>pig_properties_list = self.pig_properties.split()<EOL>pig_cmd.extend(pig_properties_list)<EOL><DEDENT>if verbose:<EOL><INDENT>self.log.info("<STR_LIT:%s>", "<STR_LIT:U+0020>".join(pig_cmd))<EOL><DEDENT>sp = subprocess.Popen(<EOL>pig_cmd,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.STDOUT,<EOL>cwd=tmp_dir,<EOL>close_fds=True)<EOL>self.sp = sp<EOL>stdout = '<STR_LIT>'<EOL>for line in iter(sp.stdout.readline, b'<STR_LIT>'):<EOL><INDENT>stdout += line.decode('<STR_LIT:utf-8>')<EOL>if verbose:<EOL><INDENT>self.log.info(line.strip())<EOL><DEDENT><DEDENT>sp.wait()<EOL>if sp.returncode:<EOL><INDENT>raise AirflowException(stdout)<EOL><DEDENT>return stdout<EOL><DEDENT><DEDENT>

Run an pig script using the pig cli

>>> ph = PigCliHook()
>>> result = ph.run_cli("ls /;")
>>> ("hdfs://" in result)
True


<EOL>effective_user = self.proxy_user<EOL>autoconfig = self.autoconfig<EOL>use_sasl = configuration.conf.get('<STR_LIT>', '<STR_LIT>') == '<STR_LIT>'<EOL>try:<EOL><INDENT>connections = self.get_connections(self.hdfs_conn_id)<EOL>if not effective_user:<EOL><INDENT>effective_user = connections[<NUM_LIT:0>].login<EOL><DEDENT>if not autoconfig:<EOL><INDENT>autoconfig = connections[<NUM_LIT:0>].extra_dejson.get('<STR_LIT>',<EOL>False)<EOL><DEDENT>hdfs_namenode_principal = connections[<NUM_LIT:0>].extra_dejson.get(<EOL>'<STR_LIT>')<EOL><DEDENT>except AirflowException:<EOL><INDENT>if not autoconfig:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>if autoconfig:<EOL><INDENT>client = AutoConfigClient(effective_user=effective_user,<EOL>use_sasl=use_sasl)<EOL><DEDENT>elif len(connections) == <NUM_LIT:1>:<EOL><INDENT>client = Client(connections[<NUM_LIT:0>].host, connections[<NUM_LIT:0>].port,<EOL>effective_user=effective_user, use_sasl=use_sasl,<EOL>hdfs_namenode_principal=hdfs_namenode_principal)<EOL><DEDENT>elif len(connections) > <NUM_LIT:1>:<EOL><INDENT>nn = [Namenode(conn.host, conn.port) for conn in connections]<EOL>client = HAClient(nn, effective_user=effective_user,<EOL>use_sasl=use_sasl,<EOL>hdfs_namenode_principal=hdfs_namenode_principal)<EOL><DEDENT>else:<EOL><INDENT>raise HDFSHookException("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>return client<EOL>


self.copy_expert("<STR_LIT>".format(table=table), tmp_file)<EOL>

Loads a tab-delimited file into a database table


return cell<EOL>

Postgresql will adapt all arguments to the execute() method internally,
hence we return cell without any conversion.

See http://initd.org/psycopg/docs/advanced.html#adapting-new-types for
more information.

:param cell: The cell to insert into the table
:type cell: object
:param conn: The database connection
:type conn: connection object
:return: The cell
:rtype: object


Dumps a database table into a tab-delimited file


retry_after = int(<EOL>rate_limit_exception.response.headers.get('<STR_LIT>', <NUM_LIT>))<EOL>self.log.info(<EOL>"<STR_LIT>",<EOL>retry_after<EOL>)<EOL>time.sleep(retry_after)<EOL>

Sleep for the time specified in the exception. If not specified, wait
for 60 seconds.


conn = self.get_connection(self.sqlite_conn_id)<EOL>conn = sqlite3.connect(conn.host)<EOL>return conn<EOL>