def read(self, task_instance, try_number=None, metadata=None):

<EOL>if try_number is None:<EOL><INDENT>next_try = task_instance.next_try_number<EOL>try_numbers = list(range(<NUM_LIT:1>, next_try))<EOL><DEDENT>elif try_number < <NUM_LIT:1>:<EOL><INDENT>logs = [<EOL>'<STR_LIT>'.format(try_number),<EOL>]<EOL>return logs<EOL><DEDENT>else:<EOL><INDENT>try_numbers = [try_number]<EOL><DEDENT>logs = ['<STR_LIT>'] * len(try_numbers)<EOL>metadatas = [{}] * len(try_numbers)<EOL>for i, try_number in enumerate(try_numbers):<EOL><INDENT>log, metadata = self._read(task_instance, try_number, metadata)<EOL>logs[i] += log<EOL>metadatas[i] = metadata<EOL><DEDENT>return logs, metadatas<EOL>

Read logs of given task instance from local machine.
:param task_instance: task instance object
:param try_number: task instance try_number to read logs from. If None
                   it returns all logs separated by try_number
:param metadata: log metadata,
                 can be used for steaming log reading and auto-tailing.
:return: a list of logs


<EOL>relative_path = self._render_filename(ti, ti.try_number)<EOL>full_path = os.path.join(self.local_base, relative_path)<EOL>directory = os.path.dirname(full_path)<EOL>if not os.path.exists(directory):<EOL><INDENT>mkdirs(directory, <NUM_LIT>)<EOL><DEDENT>if not os.path.exists(full_path):<EOL><INDENT>open(full_path, "<STR_LIT:a>").close()<EOL>os.chmod(full_path, <NUM_LIT>)<EOL><DEDENT>return full_path<EOL>

Create log directory and give it correct permissions.
:param ti: task instance object
:return: relative log path of the given task instance


def __init__(self, base_log_folder, filename_template):

super().__init__()<EOL>self.handler = None<EOL>self.base_log_folder = base_log_folder<EOL>self.dag_dir = os.path.expanduser(settings.DAGS_FOLDER)<EOL>self.filename_template, self.filename_jinja_template =parse_template_string(filename_template)<EOL>self._cur_date = datetime.today()<EOL>if not os.path.exists(self._get_log_directory()):<EOL><INDENT>try:<EOL><INDENT>os.makedirs(self._get_log_directory())<EOL><DEDENT>except OSError:<EOL><INDENT>if not os.path.isdir(self._get_log_directory()):<EOL><INDENT>raise<EOL><DEDENT>logging.warning("<STR_LIT>", self._get_log_directory())<EOL><DEDENT><DEDENT>self._symlink_latest_log_directory()<EOL>

:param base_log_folder: Base log folder to place logs.
:param filename_template: template filename string


local_loc = self._init_file(filename)<EOL>self.handler = logging.FileHandler(local_loc)<EOL>self.handler.setFormatter(self.formatter)<EOL>self.handler.setLevel(self.level)<EOL>if self._cur_date < datetime.today():<EOL><INDENT>self._symlink_latest_log_directory()<EOL>self._cur_date = datetime.today()<EOL><DEDENT>

Provide filename context to airflow task handler.
:param filename: filename in which the dag is located


relative_path = self._render_filename(filename)<EOL>full_path = os.path.join(self._get_log_directory(), relative_path)<EOL>directory = os.path.dirname(full_path)<EOL>if not os.path.exists(directory):<EOL><INDENT>try:<EOL><INDENT>os.makedirs(directory)<EOL><DEDENT>except OSError:<EOL><INDENT>if not os.path.isdir(directory):<EOL><INDENT>raise<EOL><DEDENT><DEDENT><DEDENT>if not os.path.exists(full_path):<EOL><INDENT>open(full_path, "<STR_LIT:a>").close()<EOL><DEDENT>return full_path<EOL>

Create log file and directory if required.
:param filename: task instance object
:return: relative log path of the given task instance


def __init__(self, base_log_folder, filename_template,<EOL>log_id_template, end_of_log_mark,<EOL>host='<STR_LIT>'):

super().__init__(<EOL>base_log_folder, filename_template)<EOL>self.closed = False<EOL>self.log_id_template, self.log_id_jinja_template =parse_template_string(log_id_template)<EOL>self.client = elasticsearch.Elasticsearch([host])<EOL>self.mark_end_on_close = True<EOL>self.end_of_log_mark = end_of_log_mark<EOL>

:param base_log_folder: base folder to store logs locally
:param log_id_template: log id template
:param host: Elasticsearch host name


def _read(self, ti, try_number, metadata=None):

if not metadata:<EOL><INDENT>metadata = {'<STR_LIT>': <NUM_LIT:0>}<EOL><DEDENT>if '<STR_LIT>' not in metadata:<EOL><INDENT>metadata['<STR_LIT>'] = <NUM_LIT:0><EOL><DEDENT>offset = metadata['<STR_LIT>']<EOL>log_id = self._render_log_id(ti, try_number)<EOL>logs = self.es_read(log_id, offset)<EOL>next_offset = offset if not logs else logs[-<NUM_LIT:1>].offset<EOL>metadata['<STR_LIT>'] = next_offset<EOL>metadata['<STR_LIT>'] = False if not logselse logs[-<NUM_LIT:1>].message == self.end_of_log_mark.strip()<EOL>cur_ts = pendulum.now()<EOL>if '<STR_LIT>' in metadata:<EOL><INDENT>last_log_ts = timezone.parse(metadata['<STR_LIT>'])<EOL>if cur_ts.diff(last_log_ts).in_minutes() >= <NUM_LIT:5>:<EOL><INDENT>metadata['<STR_LIT>'] = True<EOL><DEDENT><DEDENT>if offset != next_offset or '<STR_LIT>' not in metadata:<EOL><INDENT>metadata['<STR_LIT>'] = str(cur_ts)<EOL><DEDENT>message = '<STR_LIT:\n>'.join([log.message for log in logs])<EOL>return message, metadata<EOL>

Endpoint for streaming log.
:param ti: task instance object
:param try_number: try_number of the task instance
:param metadata: log metadata,
                 can be used for steaming log reading and auto-tailing.
:return: a list of log documents and metadata.


<EOL>s = Search(using=self.client).query('<STR_LIT>', log_id=log_id).sort('<STR_LIT>')<EOL>s = s.filter('<STR_LIT>', offset={'<STR_LIT>': offset})<EOL>logs = []<EOL>if s.count() != <NUM_LIT:0>:<EOL><INDENT>try:<EOL><INDENT>logs = s[self.MAX_LINE_PER_PAGE * self.PAGE:self.MAX_LINE_PER_PAGE].execute()<EOL><DEDENT>except Exception as e:<EOL><INDENT>self.log.exception('<STR_LIT>', log_id, str(e))<EOL><DEDENT><DEDENT>return logs<EOL>

Returns the logs matching log_id in Elasticsearch and next offset.
Returns '' if no log is found or there was an error.
:param log_id: the log_id of the log to read.
:type log_id: str
:param offset: the offset start to read log from.
:type offset: str


<EOL>if self.closed:<EOL><INDENT>return<EOL><DEDENT>super().close()<EOL>if not self.upload_on_close:<EOL><INDENT>return<EOL><DEDENT>local_loc = os.path.join(self.local_base, self.log_relative_path)<EOL>remote_loc = os.path.join(self.remote_base, self.log_relative_path)<EOL>if os.path.exists(local_loc):<EOL><INDENT>with open(local_loc, '<STR_LIT:r>') as logfile:<EOL><INDENT>log = logfile.read()<EOL><DEDENT>self.gcs_write(log, remote_loc)<EOL><DEDENT>self.closed = True<EOL>

Close and upload local log file to remote storage S3.


<EOL>log_relative_path = self._render_filename(ti, try_number)<EOL>remote_loc = os.path.join(self.remote_base, log_relative_path)<EOL>try:<EOL><INDENT>remote_log = self.gcs_read(remote_loc)<EOL>log = '<STR_LIT>'.format(<EOL>remote_loc, remote_log)<EOL>return log, {'<STR_LIT>': True}<EOL><DEDENT>except Exception as e:<EOL><INDENT>log = '<STR_LIT>'.format(<EOL>remote_loc, str(e))<EOL>self.log.error(log)<EOL>local_log, metadata = super()._read(ti, try_number)<EOL>log += local_log<EOL>return log, metadata<EOL><DEDENT>

Read logs of given task instance and try_number from GCS.
If failed, read the log from task instance host machine.
:param ti: task instance object
:param try_number: task instance try_number to read logs from
:param metadata: log metadata,
                 can be used for steaming log reading and auto-tailing.


bkt, blob = self.parse_gcs_url(remote_log_location)<EOL>return self.hook.download(bkt, blob).decode('<STR_LIT:utf-8>')<EOL>

Returns the log found at the remote_log_location.
:param remote_log_location: the log's location in remote storage
:type remote_log_location: str (path)


def gcs_write(self, log, remote_log_location, append=True):

if append:<EOL><INDENT>try:<EOL><INDENT>old_log = self.gcs_read(remote_log_location)<EOL>log = '<STR_LIT:\n>'.join([old_log, log]) if old_log else log<EOL><DEDENT>except Exception as e:<EOL><INDENT>if not hasattr(e, '<STR_LIT>') or e.resp.get('<STR_LIT:status>') != '<STR_LIT>':<EOL><INDENT>log = '<STR_LIT>'.format(str(e)) + log<EOL><DEDENT><DEDENT><DEDENT>try:<EOL><INDENT>bkt, blob = self.parse_gcs_url(remote_log_location)<EOL>from tempfile import NamedTemporaryFile<EOL>with NamedTemporaryFile(mode='<STR_LIT>') as tmpfile:<EOL><INDENT>tmpfile.write(log)<EOL>tmpfile.flush()<EOL>self.hook.upload(bkt, blob, tmpfile.name)<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>self.log.error('<STR_LIT>', remote_log_location, e)<EOL><DEDENT>

Writes the log to the remote_log_location. Fails silently if no hook
was created.
:param log: the log to write to the remote_log_location
:type log: str
:param remote_log_location: the log's location in remote storage
:type remote_log_location: str (path)
:param append: if False, any existing log file is overwritten. If True,
    the new log is appended to any existing logs.
:type append: bool


parsed_url = urlparse(gsurl)<EOL>if not parsed_url.netloc:<EOL><INDENT>raise AirflowException('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>bucket = parsed_url.netloc<EOL>blob = parsed_url.path.strip('<STR_LIT:/>')<EOL>return bucket, blob<EOL><DEDENT>

Given a Google Cloud Storage URL (gs://<bucket>/<blob>), returns a
tuple containing the corresponding bucket and blob.


def process_result_value(self, value, dialect):

if value is not None:<EOL><INDENT>if value.tzinfo is None:<EOL><INDENT>value = value.replace(tzinfo=utc)<EOL><DEDENT>else:<EOL><INDENT>value = value.astimezone(utc)<EOL><DEDENT><DEDENT>return value<EOL>

Processes DateTimes from the DB making sure it is always
returning UTC. Not using timezone.convert_to_utc as that
converts to configured TIMEZONE while the DB might be
running with some other setting. We assume UTC datetimes
in the database.


@functools.wraps(f)<EOL>def wrapper(*args, **kwargs):<EOL><INDENT>"""<STR_LIT>"""<EOL>assert args<EOL>assert isinstance(args[<NUM_LIT:0>], Namespace),"<STR_LIT>""<STR_LIT>".format(args[<NUM_LIT:0>])<EOL>metrics = _build_metrics(f.__name__, args[<NUM_LIT:0>])<EOL>cli_action_loggers.on_pre_execution(**metrics)<EOL>try:<EOL><INDENT>return f(*args, **kwargs)<EOL><DEDENT>except Exception as e:<EOL><INDENT>metrics['<STR_LIT:error>'] = e<EOL>raise<EOL><DEDENT>finally:<EOL><INDENT>metrics['<STR_LIT>'] = datetime.utcnow()<EOL>cli_action_loggers.on_post_execution(**metrics)<EOL><DEDENT><DEDENT>return wrapper<EOL>

Decorates function to execute function at the same time submitting action_logging
but in CLI context. It will call action logger callbacks twice,
one for pre-execution and the other one for post-execution.

Action logger will be called with below keyword parameters:
    sub_command : name of sub-command
    start_datetime : start datetime instance by utc
    end_datetime : end datetime instance by utc
    full_command : full command line arguments
    user : current user
    log : airflow.models.log.Log ORM instance
    dag_id : dag id (optional)
    task_id : task_id (optional)
    execution_date : execution date (optional)
    error : exception instance if there's an exception

:param f: function instance
:return: wrapped function


metrics = {'<STR_LIT>': func_name, '<STR_LIT>': datetime.utcnow(),<EOL>'<STR_LIT>': '<STR_LIT:{}>'.format(list(sys.argv)), '<STR_LIT:user>': getpass.getuser()}<EOL>assert isinstance(namespace, Namespace)<EOL>tmp_dic = vars(namespace)<EOL>metrics['<STR_LIT>'] = tmp_dic.get('<STR_LIT>')<EOL>metrics['<STR_LIT>'] = tmp_dic.get('<STR_LIT>')<EOL>metrics['<STR_LIT>'] = tmp_dic.get('<STR_LIT>')<EOL>metrics['<STR_LIT>'] = socket.gethostname()<EOL>extra = json.dumps(dict((k, metrics[k]) for k in ('<STR_LIT>', '<STR_LIT>')))<EOL>log = Log(<EOL>event='<STR_LIT>'.format(func_name),<EOL>task_instance=None,<EOL>owner=metrics['<STR_LIT:user>'],<EOL>extra=extra,<EOL>task_id=metrics.get('<STR_LIT>'),<EOL>dag_id=metrics.get('<STR_LIT>'),<EOL>execution_date=metrics.get('<STR_LIT>'))<EOL>metrics['<STR_LIT>'] = log<EOL>return metrics<EOL>

Builds metrics dict from function args
It assumes that function arguments is from airflow.bin.cli module's function
and has Namespace instance where it optionally contains "dag_id", "task_id",
and "execution_date".

:param func_name: name of function
:param namespace: Namespace instance from argparse
:return: dict with metrics


try:<EOL><INDENT>module_path, class_name = dotted_path.rsplit('<STR_LIT:.>', <NUM_LIT:1>)<EOL><DEDENT>except ValueError:<EOL><INDENT>raise ImportError("<STR_LIT>".format(dotted_path))<EOL><DEDENT>module = import_module(module_path)<EOL>try:<EOL><INDENT>return getattr(module, class_name)<EOL><DEDENT>except AttributeError:<EOL><INDENT>raise ImportError('<STR_LIT>'.format(<EOL>module_path, class_name)<EOL>)<EOL><DEDENT>

Import a dotted module path and return the attribute/class designated by the
last name in the path. Raise ImportError if the import failed.


<EOL>try:<EOL><INDENT>callable_path = conf.get('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>except AirflowConfigException:<EOL><INDENT>callable_path = None<EOL><DEDENT>if not callable_path:<EOL><INDENT>return socket.getfqdn()<EOL><DEDENT>module_path, attr_name = callable_path.split('<STR_LIT::>')<EOL>module = importlib.import_module(module_path)<EOL>callable = getattr(module, attr_name)<EOL>return callable()<EOL>

Fetch the hostname using the callable from the config or using
`socket.getfqdn` as a fallback.


def date_range(start_date, end_date=None, num=None, delta=None):

if not delta:<EOL><INDENT>return []<EOL><DEDENT>if end_date and start_date > end_date:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if end_date and num:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if not end_date and not num:<EOL><INDENT>end_date = timezone.utcnow()<EOL><DEDENT>delta_iscron = False<EOL>tz = start_date.tzinfo<EOL>if isinstance(delta, six.string_types):<EOL><INDENT>delta_iscron = True<EOL>start_date = timezone.make_naive(start_date, tz)<EOL>cron = croniter(delta, start_date)<EOL><DEDENT>elif isinstance(delta, timedelta):<EOL><INDENT>delta = abs(delta)<EOL><DEDENT>dates = []<EOL>if end_date:<EOL><INDENT>if timezone.is_naive(start_date):<EOL><INDENT>end_date = timezone.make_naive(end_date, tz)<EOL><DEDENT>while start_date <= end_date:<EOL><INDENT>if timezone.is_naive(start_date):<EOL><INDENT>dates.append(timezone.make_aware(start_date, tz))<EOL><DEDENT>else:<EOL><INDENT>dates.append(start_date)<EOL><DEDENT>if delta_iscron:<EOL><INDENT>start_date = cron.get_next(datetime)<EOL><DEDENT>else:<EOL><INDENT>start_date += delta<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for _ in range(abs(num)):<EOL><INDENT>if timezone.is_naive(start_date):<EOL><INDENT>dates.append(timezone.make_aware(start_date, tz))<EOL><DEDENT>else:<EOL><INDENT>dates.append(start_date)<EOL><DEDENT>if delta_iscron:<EOL><INDENT>if num > <NUM_LIT:0>:<EOL><INDENT>start_date = cron.get_next(datetime)<EOL><DEDENT>else:<EOL><INDENT>start_date = cron.get_prev(datetime)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if num > <NUM_LIT:0>:<EOL><INDENT>start_date += delta<EOL><DEDENT>else:<EOL><INDENT>start_date -= delta<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return sorted(dates)<EOL>

Get a set of dates as a list based on a start, end and delta, delta
can be something that can be added to `datetime.datetime`
or a cron expression as a `str`

:Example::

    date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta=timedelta(1))
        [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0),
        datetime.datetime(2016, 1, 3, 0, 0)]
    date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta='0 0 * * *')
        [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0),
        datetime.datetime(2016, 1, 3, 0, 0)]
    date_range(datetime(2016, 1, 1), datetime(2016, 3, 3), delta="0 0 0 * *")
        [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 2, 1, 0, 0),
        datetime.datetime(2016, 3, 1, 0, 0)]

:param start_date: anchor date to start the series from
:type start_date: datetime.datetime
:param end_date: right boundary for the date range
:type end_date: datetime.datetime
:param num: alternatively to end_date, you can specify the number of
    number of entries you want in the range. This number can be negative,
    output will always be sorted regardless
:type num: int


def round_time(dt, delta, start_date=timezone.make_aware(datetime.min)):

if isinstance(delta, six.string_types):<EOL><INDENT>tz = start_date.tzinfo<EOL>start_date = timezone.make_naive(start_date, tz)<EOL>cron = croniter(delta, start_date)<EOL>prev = cron.get_prev(datetime)<EOL>if prev == start_date:<EOL><INDENT>return timezone.make_aware(start_date, tz)<EOL><DEDENT>else:<EOL><INDENT>return timezone.make_aware(prev, tz)<EOL><DEDENT><DEDENT>dt -= timedelta(microseconds=dt.microsecond)<EOL>upper = <NUM_LIT:1><EOL>while start_date + upper * delta < dt:<EOL><INDENT>upper *= <NUM_LIT:2><EOL><DEDENT>lower = upper // <NUM_LIT:2><EOL>while True:<EOL><INDENT>if start_date + (lower + <NUM_LIT:1>) * delta >= dt:<EOL><INDENT>if (start_date + (lower + <NUM_LIT:1>) * delta) - dt <= dt - (start_date + lower * delta):<EOL><INDENT>return start_date + (lower + <NUM_LIT:1>) * delta<EOL><DEDENT>else:<EOL><INDENT>return start_date + lower * delta<EOL><DEDENT><DEDENT>candidate = lower + (upper - lower) // <NUM_LIT:2><EOL>if start_date + candidate * delta >= dt:<EOL><INDENT>upper = candidate<EOL><DEDENT>else:<EOL><INDENT>lower = candidate<EOL><DEDENT><DEDENT>

Returns the datetime of the form start_date + i * delta
which is closest to dt for any non-negative integer i.
Note that delta may be a datetime.timedelta or a dateutil.relativedelta
>>> round_time(datetime(2015, 1, 1, 6), timedelta(days=1))
datetime.datetime(2015, 1, 1, 0, 0)
>>> round_time(datetime(2015, 1, 2), relativedelta(months=1))
datetime.datetime(2015, 1, 1, 0, 0)
>>> round_time(datetime(2015, 9, 16, 0, 0), timedelta(1), datetime(2015, 9, 14, 0, 0))
datetime.datetime(2015, 9, 16, 0, 0)
>>> round_time(datetime(2015, 9, 15, 0, 0), timedelta(1), datetime(2015, 9, 14, 0, 0))
datetime.datetime(2015, 9, 15, 0, 0)
>>> round_time(datetime(2015, 9, 14, 0, 0), timedelta(1), datetime(2015, 9, 14, 0, 0))
datetime.datetime(2015, 9, 14, 0, 0)
>>> round_time(datetime(2015, 9, 13, 0, 0), timedelta(1), datetime(2015, 9, 14, 0, 0))
datetime.datetime(2015, 9, 14, 0, 0)


if len(time_seconds_arr) == <NUM_LIT:0>:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>max_time_seconds = max(time_seconds_arr)<EOL>if max_time_seconds <= <NUM_LIT> * <NUM_LIT:2>:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>elif max_time_seconds <= <NUM_LIT> * <NUM_LIT> * <NUM_LIT:2>:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>elif max_time_seconds <= <NUM_LIT> * <NUM_LIT> * <NUM_LIT> * <NUM_LIT:2>:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>

Determine the most appropriate time unit for an array of time durations
specified in seconds.
e.g. 5400 seconds => 'minutes', 36000 seconds => 'hours'


if unit == '<STR_LIT>':<EOL><INDENT>return list(map(lambda x: x * <NUM_LIT:1.0> / <NUM_LIT>, time_seconds_arr))<EOL><DEDENT>elif unit == '<STR_LIT>':<EOL><INDENT>return list(map(lambda x: x * <NUM_LIT:1.0> / (<NUM_LIT> * <NUM_LIT>), time_seconds_arr))<EOL><DEDENT>elif unit == '<STR_LIT>':<EOL><INDENT>return list(map(lambda x: x * <NUM_LIT:1.0> / (<NUM_LIT> * <NUM_LIT> * <NUM_LIT>), time_seconds_arr))<EOL><DEDENT>return time_seconds_arr<EOL>

Convert an array of time durations in seconds to the specified time unit.


def days_ago(n, hour=<NUM_LIT:0>, minute=<NUM_LIT:0>, second=<NUM_LIT:0>, microsecond=<NUM_LIT:0>):

today = timezone.utcnow().replace(<EOL>hour=hour,<EOL>minute=minute,<EOL>second=second,<EOL>microsecond=microsecond)<EOL>return today - timedelta(days=n)<EOL>

Get a datetime object representing `n` days ago. By default the time is
set to midnight.


return timezone.parse(execution_date_str)<EOL>

Parse execution date string to datetime object.


def send_email(to, subject, html_content,<EOL>files=None, dryrun=False, cc=None, bcc=None,<EOL>mime_subtype='<STR_LIT>', mime_charset='<STR_LIT:utf-8>', **kwargs):

path, attr = configuration.conf.get('<STR_LIT:email>', '<STR_LIT>').rsplit('<STR_LIT:.>', <NUM_LIT:1>)<EOL>module = importlib.import_module(path)<EOL>backend = getattr(module, attr)<EOL>to = get_email_address_list(to)<EOL>to = "<STR_LIT:U+002CU+0020>".join(to)<EOL>return backend(to, subject, html_content, files=files,<EOL>dryrun=dryrun, cc=cc, bcc=bcc,<EOL>mime_subtype=mime_subtype, mime_charset=mime_charset, **kwargs)<EOL>

Send email using backend specified in EMAIL_BACKEND.


def send_email_smtp(to, subject, html_content, files=None,<EOL>dryrun=False, cc=None, bcc=None,<EOL>mime_subtype='<STR_LIT>', mime_charset='<STR_LIT:utf-8>',<EOL>**kwargs):

smtp_mail_from = configuration.conf.get('<STR_LIT>', '<STR_LIT>')<EOL>to = get_email_address_list(to)<EOL>msg = MIMEMultipart(mime_subtype)<EOL>msg['<STR_LIT>'] = subject<EOL>msg['<STR_LIT>'] = smtp_mail_from<EOL>msg['<STR_LIT>'] = "<STR_LIT:U+002CU+0020>".join(to)<EOL>recipients = to<EOL>if cc:<EOL><INDENT>cc = get_email_address_list(cc)<EOL>msg['<STR_LIT>'] = "<STR_LIT:U+002CU+0020>".join(cc)<EOL>recipients = recipients + cc<EOL><DEDENT>if bcc:<EOL><INDENT>bcc = get_email_address_list(bcc)<EOL>recipients = recipients + bcc<EOL><DEDENT>msg['<STR_LIT>'] = formatdate(localtime=True)<EOL>mime_text = MIMEText(html_content, '<STR_LIT:html>', mime_charset)<EOL>msg.attach(mime_text)<EOL>for fname in files or []:<EOL><INDENT>basename = os.path.basename(fname)<EOL>with open(fname, "<STR_LIT:rb>") as f:<EOL><INDENT>part = MIMEApplication(<EOL>f.read(),<EOL>Name=basename<EOL>)<EOL>part['<STR_LIT>'] = '<STR_LIT>' % basename<EOL>part['<STR_LIT>'] = '<STR_LIT>' % basename<EOL>msg.attach(part)<EOL><DEDENT><DEDENT>send_MIME_email(smtp_mail_from, recipients, msg, dryrun)<EOL>

Send an email with html content

>>> send_email('test@example.com', 'foo', '<b>Foo</b> bar', ['/dev/null'], dryrun=True)


cfg_dict = conf.as_dict(display_sensitive=True, raw=True)<EOL>temp_fd, cfg_path = mkstemp()<EOL>with os.fdopen(temp_fd, '<STR_LIT:w>') as temp_file:<EOL><INDENT>if chmod is not None:<EOL><INDENT>os.fchmod(temp_fd, chmod)<EOL><DEDENT>json.dump(cfg_dict, temp_file)<EOL><DEDENT>return cfg_path<EOL>

Returns a path for a temporary file including a full copy of the configuration
settings.
:return: a path to a temporary file


try:<EOL><INDENT>o_umask = os.umask(<NUM_LIT:0>)<EOL>os.makedirs(path, mode)<EOL><DEDENT>except OSError:<EOL><INDENT>if not os.path.isdir(path):<EOL><INDENT>raise<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>os.umask(o_umask)<EOL><DEDENT>

Creates the directory specified by path, creating intermediate directories
as necessary. If directory already exists, this is a no-op.

:param path: The directory to create
:type path: str
:param mode: The mode to give to the directory e.g. 0o755, ignores umask
:type mode: int


if not obj:<EOL><INDENT>return None<EOL><DEDENT>d = {}<EOL>for c in obj.__table__.columns:<EOL><INDENT>value = getattr(obj, c.name)<EOL>if type(value) == datetime:<EOL><INDENT>value = value.isoformat()<EOL><DEDENT>d[c.name] = value<EOL><DEDENT>return d<EOL>

Transforms a SQLAlchemy model instance into a dictionary


for item in l:<EOL><INDENT>if item is obj:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Checks whether an object is one of the item in the list.
This is different from ``in`` because ``in`` uses __cmp__ when
present. Here we change based on the object itself


return hasattr(obj, '<STR_LIT>') and not isinstance(obj, basestring)<EOL>

Test if an object is a container (iterable) but not a string


if is_container(obj):<EOL><INDENT>return tuple(obj)<EOL><DEDENT>else:<EOL><INDENT>return tuple([obj])<EOL><DEDENT>

If obj is a container, returns obj as a tuple.
Otherwise, returns a tuple containing obj.


if chunk_size <= <NUM_LIT:0>:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>for i in range(<NUM_LIT:0>, len(items), chunk_size):<EOL><INDENT>yield items[i:i + chunk_size]<EOL><DEDENT>

Yield successive chunks of a given size from a list of items


def reduce_in_chunks(fn, iterable, initializer, chunk_size=<NUM_LIT:0>):

if len(iterable) == <NUM_LIT:0>:<EOL><INDENT>return initializer<EOL><DEDENT>if chunk_size == <NUM_LIT:0>:<EOL><INDENT>chunk_size = len(iterable)<EOL><DEDENT>return reduce(fn, chunks(iterable, chunk_size), initializer)<EOL>

Reduce the given list of items by splitting it into chunks
of the given size and passing each chunk through the reducer


return [e for i in iterable for e in i]<EOL>

Return an iterable with one level flattened

>>> as_flattened_list((('blue', 'red'), ('green', 'yellow', 'pink')))
['blue', 'red', 'green', 'yellow', 'pink']


for up_task, down_task in zip(tasks[:-<NUM_LIT:1>], tasks[<NUM_LIT:1>:]):<EOL><INDENT>up_task.set_downstream(down_task)<EOL><DEDENT>

Given a number of tasks, builds a dependency chain.

chain(task_1, task_2, task_3, task_4)

is equivalent to

task_1.set_downstream(task_2)
task_2.set_downstream(task_3)
task_3.set_downstream(task_4)


for task in from_tasks:<EOL><INDENT>task.set_downstream(to_tasks)<EOL><DEDENT>

r"""
    Set downstream dependencies for all tasks in from_tasks to all tasks in to_tasks.
    E.g.: cross_downstream(from_tasks=[t1, t2, t3], to_tasks=[t4, t5, t6])
    Is equivalent to:

    t1 --> t4
       \ /
    t2 -X> t5
       / \
    t3 --> t6

    t1.set_downstream(t4)
    t1.set_downstream(t5)
    t1.set_downstream(t6)
    t2.set_downstream(t4)
    t2.set_downstream(t5)
    t2.set_downstream(t6)
    t3.set_downstream(t4)
    t3.set_downstream(t5)
    t3.set_downstream(t6)

    :param from_tasks: List of tasks to start from.
    :type from_tasks: List[airflow.models.BaseOperator]
    :param to_tasks: List of tasks to set as downstream dependencies.
    :type to_tasks: List[airflow.models.BaseOperator]


if not rows:<EOL><INDENT>return<EOL><DEDENT>if hasattr(rows[<NUM_LIT:0>], '<STR_LIT>'):  <EOL><INDENT>headers = rows[<NUM_LIT:0>]._fields<EOL><DEDENT>else:<EOL><INDENT>headers = ["<STR_LIT>".format(i) for i in range(len(rows[<NUM_LIT:0>]))]<EOL><DEDENT>lens = [len(s) for s in headers]<EOL>for row in rows:<EOL><INDENT>for i in range(len(rows[<NUM_LIT:0>])):<EOL><INDENT>slenght = len("<STR_LIT:{}>".format(row[i]))<EOL>if slenght > lens[i]:<EOL><INDENT>lens[i] = slenght<EOL><DEDENT><DEDENT><DEDENT>formats = []<EOL>hformats = []<EOL>for i in range(len(rows[<NUM_LIT:0>])):<EOL><INDENT>if isinstance(rows[<NUM_LIT:0>][i], int):<EOL><INDENT>formats.append("<STR_LIT>" % lens[i])<EOL><DEDENT>else:<EOL><INDENT>formats.append("<STR_LIT>" % lens[i])<EOL><DEDENT>hformats.append("<STR_LIT>" % lens[i])<EOL><DEDENT>pattern = "<STR_LIT>".join(formats)<EOL>hpattern = "<STR_LIT>".join(hformats)<EOL>separator = "<STR_LIT>".join(['<STR_LIT:->' * n for n in lens])<EOL>s = "<STR_LIT>"<EOL>s += separator + '<STR_LIT:\n>'<EOL>s += (hpattern % tuple(headers)) + '<STR_LIT:\n>'<EOL>s += separator + '<STR_LIT:\n>'<EOL>def f(t):<EOL><INDENT>return "<STR_LIT:{}>".format(t) if isinstance(t, basestring) else t<EOL><DEDENT>for line in rows:<EOL><INDENT>s += pattern % tuple(f(t) for t in line) + '<STR_LIT:\n>'<EOL><DEDENT>s += separator + '<STR_LIT:\n>'<EOL>return s<EOL>

Returns a pretty ascii table from tuples

    If namedtuple are used, the table will have headers


def reap_process_group(pid, log, sig=signal.SIGTERM,<EOL>timeout=DEFAULT_TIME_TO_WAIT_AFTER_SIGTERM):

def on_terminate(p):<EOL><INDENT>log.info("<STR_LIT>", p, p.pid, p.returncode)<EOL><DEDENT>if pid == os.getpid():<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>parent = psutil.Process(pid)<EOL>children = parent.children(recursive=True)<EOL>children.append(parent)<EOL>try:<EOL><INDENT>pg = os.getpgid(pid)<EOL><DEDENT>except OSError as err:<EOL><INDENT>if err.errno == errno.ESRCH:<EOL><INDENT>return<EOL><DEDENT>raise<EOL><DEDENT>log.info("<STR_LIT>", sig, pg)<EOL>os.killpg(os.getpgid(pid), sig)<EOL>gone, alive = psutil.wait_procs(children, timeout=timeout, callback=on_terminate)<EOL>if alive:<EOL><INDENT>for p in alive:<EOL><INDENT>log.warn("<STR_LIT>", p, pid)<EOL><DEDENT>os.killpg(os.getpgid(pid), signal.SIGKILL)<EOL>gone, alive = psutil.wait_procs(alive, timeout=timeout, callback=on_terminate)<EOL>if alive:<EOL><INDENT>for p in alive:<EOL><INDENT>log.error("<STR_LIT>", p, p.pid)<EOL><DEDENT><DEDENT><DEDENT>

Tries really hard to terminate all children (including grandchildren). Will send
sig (SIGTERM) to the process group of pid. If any process is alive after timeout
a SIGKILL will be send.

:param log: log handler
:param pid: pid to kill
:param sig: signal type
:param timeout: how much time a process has to terminate


def render_log_filename(ti, try_number, filename_template):

filename_template, filename_jinja_template = parse_template_string(filename_template)<EOL>if filename_jinja_template:<EOL><INDENT>jinja_context = ti.get_template_context()<EOL>jinja_context['<STR_LIT>'] = try_number<EOL>return filename_jinja_template.render(**jinja_context)<EOL><DEDENT>return filename_template.format(dag_id=ti.dag_id,<EOL>task_id=ti.task_id,<EOL>execution_date=ti.execution_date.isoformat(),<EOL>try_number=try_number)<EOL>

Given task instance, try_number, filename_template, return the rendered log
filename

:param ti: task instance
:param try_number: try_number of the task
:param filename_template: filename template, which can be jinja template or
    python string template


session = settings.Session()<EOL>try:<EOL><INDENT>yield session<EOL>session.commit()<EOL><DEDENT>except Exception:<EOL><INDENT>session.rollback()<EOL>raise<EOL><DEDENT>finally:<EOL><INDENT>session.close()<EOL><DEDENT>

Contextmanager that will create and teardown a session.


@wraps(func)<EOL>def wrapper(*args, **kwargs):<EOL><INDENT>arg_session = '<STR_LIT>'<EOL>func_params = func.__code__.co_varnames<EOL>session_in_args = arg_session in func_params andfunc_params.index(arg_session) < len(args)<EOL>session_in_kwargs = arg_session in kwargs<EOL>if session_in_kwargs or session_in_args:<EOL><INDENT>return func(*args, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>with create_session() as session:<EOL><INDENT>kwargs[arg_session] = session<EOL>return func(*args, **kwargs)<EOL><DEDENT><DEDENT><DEDENT>return wrapper<EOL>

Function decorator that provides a session if it isn't provided.
If you want to reuse a session or run the function as part of a
database transaction, you pass it to the function, if not this wrapper
will create one and close it for you.


from airflow import models<EOL>from alembic.migration import MigrationContext<EOL>log.info("<STR_LIT>")<EOL>models.base.Base.metadata.drop_all(settings.engine)<EOL>mc = MigrationContext.configure(settings.engine)<EOL>if mc._version.exists(settings.engine):<EOL><INDENT>mc._version.drop(settings.engine)<EOL><DEDENT>from flask_appbuilder.models.sqla import Base<EOL>Base.metadata.drop_all(settings.engine)<EOL>initdb()<EOL>


_, archive, filename = re.search(<EOL>r'<STR_LIT>'.format(re.escape(os.sep)), fileloc).groups()<EOL>if archive and zipfile.is_zipfile(archive):<EOL><INDENT>return archive<EOL><DEDENT>else:<EOL><INDENT>return fileloc<EOL><DEDENT>

If the path contains a folder with a .zip suffix, then
the folder is treated as a zip archive and path to zip is returned.


if include_examples is None:<EOL><INDENT>include_examples = conf.getboolean('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>file_paths = []<EOL>if directory is None:<EOL><INDENT>return []<EOL><DEDENT>elif os.path.isfile(directory):<EOL><INDENT>return [directory]<EOL><DEDENT>elif os.path.isdir(directory):<EOL><INDENT>patterns_by_dir = {}<EOL>for root, dirs, files in os.walk(directory, followlinks=True):<EOL><INDENT>patterns = patterns_by_dir.get(root, [])<EOL>ignore_file = os.path.join(root, '<STR_LIT>')<EOL>if os.path.isfile(ignore_file):<EOL><INDENT>with open(ignore_file, '<STR_LIT:r>') as f:<EOL><INDENT>patterns += [re.compile(p) for p in f.read().split('<STR_LIT:\n>') if p]<EOL><DEDENT><DEDENT>dirs[:] = [<EOL>d<EOL>for d in dirs<EOL>if not any(p.search(os.path.join(root, d)) for p in patterns)<EOL>]<EOL>for d in dirs:<EOL><INDENT>patterns_by_dir[os.path.join(root, d)] = patterns<EOL><DEDENT>for f in files:<EOL><INDENT>try:<EOL><INDENT>file_path = os.path.join(root, f)<EOL>if not os.path.isfile(file_path):<EOL><INDENT>continue<EOL><DEDENT>mod_name, file_ext = os.path.splitext(<EOL>os.path.split(file_path)[-<NUM_LIT:1>])<EOL>if file_ext != '<STR_LIT>' and not zipfile.is_zipfile(file_path):<EOL><INDENT>continue<EOL><DEDENT>if any([re.findall(p, file_path) for p in patterns]):<EOL><INDENT>continue<EOL><DEDENT>might_contain_dag = True<EOL>if safe_mode and not zipfile.is_zipfile(file_path):<EOL><INDENT>with open(file_path, '<STR_LIT:rb>') as fp:<EOL><INDENT>content = fp.read()<EOL>might_contain_dag = all(<EOL>[s in content for s in (b'<STR_LIT>', b'<STR_LIT>')])<EOL><DEDENT><DEDENT>if not might_contain_dag:<EOL><INDENT>continue<EOL><DEDENT>file_paths.append(file_path)<EOL><DEDENT>except Exception:<EOL><INDENT>log = LoggingMixin().log<EOL>log.exception("<STR_LIT>", f)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if include_examples:<EOL><INDENT>import airflow.example_dags<EOL>example_dag_folder = airflow.example_dags.__path__[<NUM_LIT:0>]<EOL>file_paths.extend(list_py_file_paths(example_dag_folder, safe_mode, False))<EOL><DEDENT>return file_paths<EOL>

Traverse a directory and look for Python files.

:param directory: the directory to traverse
:type directory: unicode
:param safe_mode: whether to use a heuristic to determine whether a file
    contains Airflow DAG definitions
:return: a list of paths to Python files in the specified directory
:rtype: list[unicode]


def __init__(self, dag, pickle_id=None):

self._dag_id = dag.dag_id<EOL>self._task_ids = [task.task_id for task in dag.tasks]<EOL>self._full_filepath = dag.full_filepath<EOL>self._is_paused = dag.is_paused<EOL>self._concurrency = dag.concurrency<EOL>self._pickle_id = pickle_id<EOL>self._task_special_args = {}<EOL>for task in dag.tasks:<EOL><INDENT>special_args = {}<EOL>if task.task_concurrency is not None:<EOL><INDENT>special_args['<STR_LIT>'] = task.task_concurrency<EOL><DEDENT>if len(special_args) > <NUM_LIT:0>:<EOL><INDENT>self._task_special_args[task.task_id] = special_args<EOL><DEDENT><DEDENT>

:param dag: the DAG
:type dag: airflow.models.DAG
:param pickle_id: ID associated with the pickled version of this DAG.
:type pickle_id: unicode


return self._dag_id<EOL>


return self._task_ids<EOL>

:return: A list of task IDs that are in this DAG
:rtype: list[unicode]


return self._full_filepath<EOL>

:return: The absolute path to the file that contains this DAG's definition
:rtype: unicode


return self._concurrency<EOL>

:return: maximum number of tasks that can run simultaneously from this DAG
:rtype: int


return self._is_paused<EOL>

:return: whether this DAG is paused or not
:rtype: bool


return self._pickle_id<EOL>

:return: The pickle ID for this DAG, if it has one. Otherwise None.
:rtype: unicode


@provide_session<EOL><INDENT>def construct_task_instance(self, session=None, lock_for_update=False):<DEDENT>

TI = airflow.models.TaskInstance<EOL>qry = session.query(TI).filter(<EOL>TI.dag_id == self._dag_id,<EOL>TI.task_id == self._task_id,<EOL>TI.execution_date == self._execution_date)<EOL>if lock_for_update:<EOL><INDENT>ti = qry.with_for_update().first()<EOL><DEDENT>else:<EOL><INDENT>ti = qry.first()<EOL><DEDENT>return ti<EOL>

Construct a TaskInstance from the database based on the primary key

:param session: DB session.
:param lock_for_update: if True, indicates that the database should
    lock the TaskInstance (issuing a FOR UPDATE clause) until the
    session is committed.


self.simple_dags = simple_dags<EOL>self.dag_id_to_simple_dag = {}<EOL>for simple_dag in simple_dags:<EOL><INDENT>self.dag_id_to_simple_dag[simple_dag.dag_id] = simple_dag<EOL><DEDENT>

Constructor.

:param simple_dags: SimpleDag objects that should be in this
:type list(airflow.utils.dag_processing.SimpleDagBag)


return self.dag_id_to_simple_dag.keys()<EOL>

:return: IDs of all the DAGs in this
:rtype: list[unicode]


if dag_id not in self.dag_id_to_simple_dag:<EOL><INDENT>raise AirflowException("<STR_LIT>".format(dag_id))<EOL><DEDENT>return self.dag_id_to_simple_dag[dag_id]<EOL>

:param dag_id: DAG ID
:type dag_id: unicode
:return: if the given DAG ID exists in the bag, return the BaseDag
corresponding to that ID. Otherwise, throw an Exception
:rtype: airflow.utils.dag_processing.SimpleDag


raise NotImplementedError()<EOL>


raise NotImplementedError()<EOL>

Terminate (and then kill) the process launched to process the file


raise NotImplementedError()<EOL>

:return: the PID of the process launched to process the given file


raise NotImplementedError()<EOL>

After the process is finished, this can be called to get the return code
:return: the exit code of the process
:rtype: int


raise NotImplementedError()<EOL>

Check if the process launched to process this file is done.
:return: whether the process is finished running
:rtype: bool


raise NotImplementedError()<EOL>

:return: result of running SchedulerJob.process_file()
:rtype: list[airflow.utils.dag_processing.SimpleDag]


raise NotImplementedError()<EOL>

:return: When this started to process the file
:rtype: datetime


raise NotImplementedError()<EOL>

:return: the path to the file that this is processing
:rtype: unicode


self._file_paths = file_paths<EOL>self._file_path_queue = []<EOL>self._dag_directory = dag_directory<EOL>self._max_runs = max_runs<EOL>self._processor_factory = processor_factory<EOL>self._async_mode = async_mode<EOL>self._processors = {}<EOL>self._last_runtime = {}<EOL>self._last_finish_time = {}<EOL>self._run_count = defaultdict(int)<EOL>self._all_pids = []<EOL>self._parent_signal_conn, self._child_signal_conn = multiprocessing.Pipe()<EOL>self._stat_queue = multiprocessing.Queue()<EOL>self._result_queue = multiprocessing.Queue()<EOL>self._process = None<EOL>self._done = False<EOL>self._all_files_processed = True<EOL>self._result_count = <NUM_LIT:0><EOL>

:param dag_directory: Directory where DAG definitions are kept. All
    files in file_paths should be under this directory
:type dag_directory: unicode
:param file_paths: list of file paths that contain DAG definitions
:type file_paths: list[unicode]
:param max_runs: The number of times to parse and schedule each file. -1
    for unlimited.
:type max_runs: int
:param processor_factory: function that creates processors for DAG
    definition files. Arguments are (dag_definition_path, log_file_path)
:type processor_factory: (unicode, unicode, list) -> (AbstractDagFileProcessor)
:param async_mode: Whether to start agent in async mode
:type async_mode: bool


Launch DagFileProcessorManager processor and start DAG parsing loop in manager.


self._parent_signal_conn.send(DagParsingSignal.AGENT_HEARTBEAT)<EOL>

Should only be used when launched DAG file processor manager in sync mode.
Send agent heartbeat signal to the manager.


while True:<EOL><INDENT>if self._parent_signal_conn.recv() == DagParsingSignal.MANAGER_DONE:<EOL><INDENT>break<EOL><DEDENT><DEDENT>

Should only be used when launched DAG file processor manager in sync mode.
Wait for done signal from the manager.


<EOL>self._sync_metadata()<EOL>self._heartbeat_manager()<EOL>simple_dags = []<EOL>if sys.platform == "<STR_LIT>":<EOL><INDENT>qsize = self._result_count<EOL><DEDENT>else:<EOL><INDENT>qsize = self._result_queue.qsize()<EOL><DEDENT>for _ in range(qsize):<EOL><INDENT>simple_dags.append(self._result_queue.get())<EOL><DEDENT>self._result_count = <NUM_LIT:0><EOL>return simple_dags<EOL>

Harvest DAG parsing results from result queue and sync metadata from stat queue.
:return: List of parsing result in SimpleDag format.


if self._process and not self._process.is_alive() and not self.done:<EOL><INDENT>self.start()<EOL><DEDENT>

Heartbeat DAG file processor and start it if it is not alive.
:return:


while not self._stat_queue.empty():<EOL><INDENT>stat = self._stat_queue.get()<EOL>self._file_paths = stat.file_paths<EOL>self._all_pids = stat.all_pids<EOL>self._done = stat.done<EOL>self._all_files_processed = stat.all_files_processed<EOL>self._result_count += stat.result_count<EOL><DEDENT>

Sync metadata from stat queue and only keep the latest stat.
:return:


self.log.info("<STR_LIT>")<EOL>self._child_signal_conn.send(DagParsingSignal.TERMINATE_MANAGER)<EOL>

Send termination signal to DAG parsing processor manager
and expect it to terminate all DAG file processors.


if not self._process:<EOL><INDENT>self.log.warn('<STR_LIT>')<EOL>return<EOL><DEDENT>this_process = psutil.Process(os.getpid())<EOL>try:<EOL><INDENT>manager_process = psutil.Process(self._process.pid)<EOL><DEDENT>except psutil.NoSuchProcess:<EOL><INDENT>self.log.info("<STR_LIT>")<EOL>return<EOL><DEDENT>if manager_process.is_running()and manager_process.pid in [x.pid for x in this_process.children()]:<EOL><INDENT>self.log.info("<STR_LIT>", manager_process.pid)<EOL>manager_process.terminate()<EOL>timeout = <NUM_LIT:5><EOL>self.log.info("<STR_LIT>", timeout)<EOL>try:<EOL><INDENT>psutil.wait_procs({manager_process}, timeout)<EOL><DEDENT>except psutil.TimeoutExpired:<EOL><INDENT>self.log.debug("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>if manager_process.is_running()and manager_process.pid in [x.pid for x in this_process.children()]:<EOL><INDENT>self.log.info("<STR_LIT>", manager_process.pid)<EOL>manager_process.kill()<EOL>manager_process.wait()<EOL><DEDENT>

Terminate (and then kill) the manager process launched.
:return:


self._file_paths = file_paths<EOL>self._file_path_queue = []<EOL>self._dag_directory = dag_directory<EOL>self._max_runs = max_runs<EOL>self._processor_factory = processor_factory<EOL>self._signal_conn = signal_conn<EOL>self._stat_queue = stat_queue<EOL>self._result_queue = result_queue<EOL>self._async_mode = async_mode<EOL>self._parallelism = conf.getint('<STR_LIT>', '<STR_LIT>')<EOL>if '<STR_LIT>' in conf.get('<STR_LIT>', '<STR_LIT>') and self._parallelism > <NUM_LIT:1>:<EOL><INDENT>self.log.error("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>self._parallelism = <NUM_LIT:1><EOL><DEDENT>self._file_process_interval = conf.getint('<STR_LIT>',<EOL>'<STR_LIT>')<EOL>self.print_stats_interval = conf.getint('<STR_LIT>',<EOL>'<STR_LIT>')<EOL>self._zombie_threshold_secs = (<EOL>conf.getint('<STR_LIT>', '<STR_LIT>'))<EOL>self._processors = {}<EOL>self._last_runtime = {}<EOL>self._last_finish_time = {}<EOL>self._last_zombie_query_time = timezone.utcnow()<EOL>self.last_dag_dir_refresh_time = timezone.utcnow()<EOL>self.last_stat_print_time = timezone.datetime(<NUM_LIT>, <NUM_LIT:1>, <NUM_LIT:1>)<EOL>self._zombie_query_interval = <NUM_LIT:10><EOL>self._run_count = defaultdict(int)<EOL>self._heart_beat_key = '<STR_LIT>'<EOL>self.dag_dir_list_interval = conf.getint('<STR_LIT>',<EOL>'<STR_LIT>')<EOL>self._log = logging.getLogger('<STR_LIT>')<EOL>signal.signal(signal.SIGINT, self._exit_gracefully)<EOL>signal.signal(signal.SIGTERM, self._exit_gracefully)<EOL>

:param dag_directory: Directory where DAG definitions are kept. All
    files in file_paths should be under this directory
:type dag_directory: unicode
:param file_paths: list of file paths that contain DAG definitions
:type file_paths: list[unicode]
:param max_runs: The number of times to parse and schedule each file. -1
    for unlimited.
:type max_runs: int
:param processor_factory: function that creates processors for DAG
    definition files. Arguments are (dag_definition_path)
:type processor_factory: (unicode, unicode, list) -> (AbstractDagFileProcessor)
:param signal_conn: connection to communicate signal with processor agent.
:type signal_conn: airflow.models.connection.Connection
:param stat_queue: the queue to use for passing back parsing stat to agent.
:type stat_queue: multiprocessing.Queue
:param result_queue: the queue to use for passing back the result to agent.
:type result_queue: multiprocessing.Queue
:param async_mode: whether to start the manager in async mode
:type async_mode: bool


def _exit_gracefully(self, signum, frame):

self.log.info("<STR_LIT>", signum)<EOL>self.terminate()<EOL>self.end()<EOL>self.log.debug("<STR_LIT>")<EOL>sys.exit(os.EX_OK)<EOL>

Helper method to clean up DAG file processors to avoid leaving orphan processes.


Use multiple processes to parse and generate tasks for the
DAGs in parallel. By processing them in separate processes,
we can get parallelism and isolation from potentially harmful
user code.


while True:<EOL><INDENT>loop_start_time = time.time()<EOL>if self._signal_conn.poll():<EOL><INDENT>agent_signal = self._signal_conn.recv()<EOL>if agent_signal == DagParsingSignal.TERMINATE_MANAGER:<EOL><INDENT>self.terminate()<EOL>break<EOL><DEDENT>elif agent_signal == DagParsingSignal.END_MANAGER:<EOL><INDENT>self.end()<EOL>sys.exit(os.EX_OK)<EOL><DEDENT><DEDENT>self._refresh_dag_dir()<EOL>simple_dags = self.heartbeat()<EOL>for simple_dag in simple_dags:<EOL><INDENT>self._result_queue.put(simple_dag)<EOL><DEDENT>self._print_stat()<EOL>all_files_processed = all(self.get_last_finish_time(x) is not None<EOL>for x in self.file_paths)<EOL>max_runs_reached = self.max_runs_reached()<EOL>dag_parsing_stat = DagParsingStat(self._file_paths,<EOL>self.get_all_pids(),<EOL>max_runs_reached,<EOL>all_files_processed,<EOL>len(simple_dags))<EOL>self._stat_queue.put(dag_parsing_stat)<EOL>if max_runs_reached:<EOL><INDENT>self.log.info("<STR_LIT>"<EOL>"<STR_LIT>", self._max_runs)<EOL>break<EOL><DEDENT>loop_duration = time.time() - loop_start_time<EOL>if loop_duration < <NUM_LIT:1>:<EOL><INDENT>sleep_length = <NUM_LIT:1> - loop_duration<EOL>self.log.debug("<STR_LIT>", sleep_length)<EOL>time.sleep(sleep_length)<EOL><DEDENT><DEDENT>

Parse DAG files repeatedly in a standalone loop.


while True:<EOL><INDENT>agent_signal = self._signal_conn.recv()<EOL>if agent_signal == DagParsingSignal.TERMINATE_MANAGER:<EOL><INDENT>self.terminate()<EOL>break<EOL><DEDENT>elif agent_signal == DagParsingSignal.END_MANAGER:<EOL><INDENT>self.end()<EOL>sys.exit(os.EX_OK)<EOL><DEDENT>elif agent_signal == DagParsingSignal.AGENT_HEARTBEAT:<EOL><INDENT>self._refresh_dag_dir()<EOL>simple_dags = self.heartbeat()<EOL>for simple_dag in simple_dags:<EOL><INDENT>self._result_queue.put(simple_dag)<EOL><DEDENT>self._print_stat()<EOL>all_files_processed = all(self.get_last_finish_time(x) is not None<EOL>for x in self.file_paths)<EOL>max_runs_reached = self.max_runs_reached()<EOL>dag_parsing_stat = DagParsingStat(self._file_paths,<EOL>self.get_all_pids(),<EOL>self.max_runs_reached(),<EOL>all_files_processed,<EOL>len(simple_dags))<EOL>self._stat_queue.put(dag_parsing_stat)<EOL>self.wait_until_finished()<EOL>self._signal_conn.send(DagParsingSignal.MANAGER_DONE)<EOL>if max_runs_reached:<EOL><INDENT>self.log.info("<STR_LIT>"<EOL>"<STR_LIT>", self._max_runs)<EOL>self._signal_conn.send(DagParsingSignal.MANAGER_DONE)<EOL>break<EOL><DEDENT><DEDENT><DEDENT>

Parse DAG files in a loop controlled by DagParsingSignal.
Actual DAG parsing loop will run once upon receiving one
agent heartbeat message and will report done when finished the loop.


elapsed_time_since_refresh = (timezone.utcnow() -<EOL>self.last_dag_dir_refresh_time).total_seconds()<EOL>if elapsed_time_since_refresh > self.dag_dir_list_interval:<EOL><INDENT>self.log.info("<STR_LIT>", self._dag_directory)<EOL>self._file_paths = list_py_file_paths(self._dag_directory)<EOL>self.last_dag_dir_refresh_time = timezone.utcnow()<EOL>self.log.info("<STR_LIT>", len(self._file_paths), self._dag_directory)<EOL>self.set_file_paths(self._file_paths)<EOL>try:<EOL><INDENT>self.log.debug("<STR_LIT>")<EOL>self.clear_nonexistent_import_errors()<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.exception("<STR_LIT>")<EOL><DEDENT><DEDENT>

Refresh file paths from dag dir if we haven't done it for too long.


if ((timezone.utcnow() - self.last_stat_print_time).total_seconds() ><EOL>self.print_stats_interval):<EOL><INDENT>if len(self._file_paths) > <NUM_LIT:0>:<EOL><INDENT>self._log_file_processing_stats(self._file_paths)<EOL><DEDENT>self.last_stat_print_time = timezone.utcnow()<EOL><DEDENT>

Occasionally print out stats about how fast the files are getting processed


query = session.query(errors.ImportError)<EOL>if self._file_paths:<EOL><INDENT>query = query.filter(<EOL>~errors.ImportError.filename.in_(self._file_paths)<EOL>)<EOL><DEDENT>query.delete(synchronize_session='<STR_LIT>')<EOL>session.commit()<EOL>

Clears import errors for files that no longer exist.

:param session: session for ORM operations
:type session: sqlalchemy.orm.session.Session


<EOL>headers = ["<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>"]<EOL>rows = []<EOL>for file_path in known_file_paths:<EOL><INDENT>last_runtime = self.get_last_runtime(file_path)<EOL>file_name = os.path.basename(file_path)<EOL>file_name = os.path.splitext(file_name)[<NUM_LIT:0>].replace(os.sep, '<STR_LIT:.>')<EOL>if last_runtime:<EOL><INDENT>Stats.gauge(<EOL>'<STR_LIT>'.format(file_name),<EOL>last_runtime<EOL>)<EOL><DEDENT>processor_pid = self.get_pid(file_path)<EOL>processor_start_time = self.get_start_time(file_path)<EOL>runtime = ((timezone.utcnow() - processor_start_time).total_seconds()<EOL>if processor_start_time else None)<EOL>last_run = self.get_last_finish_time(file_path)<EOL>if last_run:<EOL><INDENT>seconds_ago = (timezone.utcnow() - last_run).total_seconds()<EOL>Stats.gauge(<EOL>'<STR_LIT>'.format(file_name),<EOL>seconds_ago<EOL>)<EOL><DEDENT>rows.append((file_path,<EOL>processor_pid,<EOL>runtime,<EOL>last_runtime,<EOL>last_run))<EOL><DEDENT>rows = sorted(rows, key=lambda x: x[<NUM_LIT:3>] or <NUM_LIT:0.0>)<EOL>formatted_rows = []<EOL>for file_path, pid, runtime, last_runtime, last_run in rows:<EOL><INDENT>formatted_rows.append((file_path,<EOL>pid,<EOL>"<STR_LIT>".format(runtime)<EOL>if runtime else None,<EOL>"<STR_LIT>".format(last_runtime)<EOL>if last_runtime else None,<EOL>last_run.strftime("<STR_LIT>")<EOL>if last_run else None))<EOL><DEDENT>log_str = ("<STR_LIT:\n>" +<EOL>"<STR_LIT:=>" * <NUM_LIT> +<EOL>"<STR_LIT:\n>" +<EOL>"<STR_LIT>" +<EOL>tabulate(formatted_rows, headers=headers) +<EOL>"<STR_LIT:\n>" +<EOL>"<STR_LIT:=>" * <NUM_LIT>)<EOL>self.log.info(log_str)<EOL>

Print out stats about how files are getting processed.

:param known_file_paths: a list of file paths that may contain Airflow
    DAG definitions
:type known_file_paths: list[unicode]
:return: None


if file_path in self._processors:<EOL><INDENT>return self._processors[file_path].pid<EOL><DEDENT>return None<EOL>

:param file_path: the path to the file that's being processed
:type file_path: unicode
:return: the PID of the process processing the given file or None if
    the specified file is not being processed
:rtype: int


return [x.pid for x in self._processors.values()]<EOL>

:return: a list of the PIDs for the processors that are running
:rtype: List[int]


if file_path in self._processors:<EOL><INDENT>return (timezone.utcnow() - self._processors[file_path].start_time).total_seconds()<EOL><DEDENT>return None<EOL>

:param file_path: the path to the file that's being processed
:type file_path: unicode
:return: the current runtime (in seconds) of the process that's
    processing the specified file or None if the file is not currently
    being processed


return self._last_runtime.get(file_path)<EOL>

:param file_path: the path to the file that was processed
:type file_path: unicode
:return: the runtime (in seconds) of the process of the last run, or
    None if the file was never processed.
:rtype: float


return self._last_finish_time.get(file_path)<EOL>

:param file_path: the path to the file that was processed
:type file_path: unicode
:return: the finish time of the process of the last run, or None if the
    file was never processed.
:rtype: datetime


if file_path in self._processors:<EOL><INDENT>return self._processors[file_path].start_time<EOL><DEDENT>return None<EOL>

:param file_path: the path to the file that's being processed
:type file_path: unicode
:return: the start time of the process that's processing the
    specified file or None if the file is not currently being processed
:rtype: datetime


self._file_paths = new_file_paths<EOL>self._file_path_queue = [x for x in self._file_path_queue<EOL>if x in new_file_paths]<EOL>filtered_processors = {}<EOL>for file_path, processor in self._processors.items():<EOL><INDENT>if file_path in new_file_paths:<EOL><INDENT>filtered_processors[file_path] = processor<EOL><DEDENT>else:<EOL><INDENT>self.log.warning("<STR_LIT>", file_path)<EOL>processor.terminate()<EOL><DEDENT><DEDENT>self._processors = filtered_processors<EOL>

Update this with a new set of paths to DAG definition files.

:param new_file_paths: list of paths to DAG definition files
:type new_file_paths: list[unicode]
:return: None


return len(self._processors)<EOL>

:return: the number of files currently being processed
:rtype: int


for file_path, processor in self._processors.items():<EOL><INDENT>while not processor.done:<EOL><INDENT>time.sleep(<NUM_LIT:0.1>)<EOL><DEDENT><DEDENT>

Sleeps until all the processors are done.


finished_processors = {}<EOL>"""<STR_LIT>"""<EOL>running_processors = {}<EOL>"""<STR_LIT>"""<EOL>for file_path, processor in self._processors.items():<EOL><INDENT>if processor.done:<EOL><INDENT>self.log.debug("<STR_LIT>", file_path)<EOL>now = timezone.utcnow()<EOL>finished_processors[file_path] = processor<EOL>self._last_runtime[file_path] = (now -<EOL>processor.start_time).total_seconds()<EOL>self._last_finish_time[file_path] = now<EOL>self._run_count[file_path] += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>running_processors[file_path] = processor<EOL><DEDENT><DEDENT>self._processors = running_processors<EOL>self.log.debug("<STR_LIT>",<EOL>len(self._processors), self._parallelism)<EOL>self.log.debug("<STR_LIT>",<EOL>len(self._file_path_queue))<EOL>simple_dags = []<EOL>for file_path, processor in finished_processors.items():<EOL><INDENT>if processor.result is None:<EOL><INDENT>self.log.warning(<EOL>"<STR_LIT>",<EOL>processor.file_path, processor.exit_code<EOL>)<EOL><DEDENT>else:<EOL><INDENT>for simple_dag in processor.result:<EOL><INDENT>simple_dags.append(simple_dag)<EOL><DEDENT><DEDENT><DEDENT>if len(self._file_path_queue) == <NUM_LIT:0>:<EOL><INDENT>file_paths_in_progress = self._processors.keys()<EOL>now = timezone.utcnow()<EOL>file_paths_recently_processed = []<EOL>for file_path in self._file_paths:<EOL><INDENT>last_finish_time = self.get_last_finish_time(file_path)<EOL>if (last_finish_time is not None and<EOL>(now - last_finish_time).total_seconds() <<EOL>self._file_process_interval):<EOL><INDENT>file_paths_recently_processed.append(file_path)<EOL><DEDENT><DEDENT>files_paths_at_run_limit = [file_path<EOL>for file_path, num_runs in self._run_count.items()<EOL>if num_runs == self._max_runs]<EOL>files_paths_to_queue = list(set(self._file_paths) -<EOL>set(file_paths_in_progress) -<EOL>set(file_paths_recently_processed) -<EOL>set(files_paths_at_run_limit))<EOL>for file_path, processor in self._processors.items():<EOL><INDENT>self.log.debug(<EOL>"<STR_LIT>",<EOL>processor.file_path, processor.start_time.isoformat()<EOL>)<EOL><DEDENT>self.log.debug(<EOL>"<STR_LIT>",<EOL>"<STR_LIT>".join(files_paths_to_queue)<EOL>)<EOL>self._file_path_queue.extend(files_paths_to_queue)<EOL><DEDENT>zombies = self._find_zombies()<EOL>while (self._parallelism - len(self._processors) > <NUM_LIT:0> and<EOL>len(self._file_path_queue) > <NUM_LIT:0>):<EOL><INDENT>file_path = self._file_path_queue.pop(<NUM_LIT:0>)<EOL>processor = self._processor_factory(file_path, zombies)<EOL>processor.start()<EOL>self.log.debug(<EOL>"<STR_LIT>",<EOL>processor.pid, file_path<EOL>)<EOL>self._processors[file_path] = processor<EOL><DEDENT>self._run_count[self._heart_beat_key] += <NUM_LIT:1><EOL>return simple_dags<EOL>

This should be periodically called by the manager loop. This method will
kick off new processes to process DAG definition files and read the
results from the finished processors.

:return: a list of SimpleDags that were produced by processors that
    have finished since the last time this was called
:rtype: list[airflow.utils.dag_processing.SimpleDag]


now = timezone.utcnow()<EOL>zombies = []<EOL>if (now - self._last_zombie_query_time).total_seconds()> self._zombie_query_interval:<EOL><INDENT>from airflow.jobs import LocalTaskJob as LJ<EOL>self.log.info("<STR_LIT>")<EOL>TI = airflow.models.TaskInstance<EOL>limit_dttm = timezone.utcnow() - timedelta(<EOL>seconds=self._zombie_threshold_secs)<EOL>self.log.info("<STR_LIT>", limit_dttm)<EOL>tis = (<EOL>session.query(TI)<EOL>.join(LJ, TI.job_id == LJ.id)<EOL>.filter(TI.state == State.RUNNING)<EOL>.filter(<EOL>or_(<EOL>LJ.state != State.RUNNING,<EOL>LJ.latest_heartbeat < limit_dttm,<EOL>)<EOL>).all()<EOL>)<EOL>self._last_zombie_query_time = timezone.utcnow()<EOL>for ti in tis:<EOL><INDENT>zombies.append(SimpleTaskInstance(ti))<EOL><DEDENT><DEDENT>return zombies<EOL>

Find zombie task instances, which are tasks haven't heartbeated for too long.
:return: Zombie task instances in SimpleTaskInstance format.


if self._max_runs == -<NUM_LIT:1>:  <EOL><INDENT>return False<EOL><DEDENT>for file_path in self._file_paths:<EOL><INDENT>if self._run_count[file_path] < self._max_runs:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>if self._run_count[self._heart_beat_key] < self._max_runs:<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

:return: whether all file paths have been processed max_runs times


for processor in self._processors.values():<EOL><INDENT>processor.terminate()<EOL><DEDENT>

Stops all running processors
:return: None


pids_to_kill = self.get_all_pids()<EOL>if len(pids_to_kill) > <NUM_LIT:0>:<EOL><INDENT>this_process = psutil.Process(os.getpid())<EOL>child_processes = [x for x in this_process.children(recursive=True)<EOL>if x.is_running() and x.pid in pids_to_kill]<EOL>for child in child_processes:<EOL><INDENT>self.log.info("<STR_LIT>", child.pid)<EOL>child.terminate()<EOL><DEDENT>timeout = <NUM_LIT:5><EOL>self.log.info("<STR_LIT>", timeout)<EOL>try:<EOL><INDENT>psutil.wait_procs(<EOL>child_processes, timeout=timeout,<EOL>callback=lambda x: self.log.info('<STR_LIT>', x.pid))<EOL><DEDENT>except psutil.TimeoutExpired:<EOL><INDENT>self.log.debug("<STR_LIT>")<EOL><DEDENT>child_processes = [x for x in this_process.children(recursive=True)<EOL>if x.is_running() and x.pid in pids_to_kill]<EOL>if len(child_processes) > <NUM_LIT:0>:<EOL><INDENT>self.log.info("<STR_LIT>")<EOL>for child in child_processes:<EOL><INDENT>self.log.info("<STR_LIT>", child.pid)<EOL>child.kill()<EOL>child.wait()<EOL><DEDENT><DEDENT><DEDENT>

Kill all child processes on exit since we don't want to leave
them as orphaned.


logging.debug("<STR_LIT>", action_logger)<EOL>__pre_exec_callbacks.append(action_logger)<EOL>

Registers more action_logger function callback for pre-execution.
This function callback is expected to be called with keyword args.
For more about the arguments that is being passed to the callback,
refer to airflow.utils.cli.action_logging()
:param action_logger: An action logger function
:return: None


logging.debug("<STR_LIT>", action_logger)<EOL>__post_exec_callbacks.append(action_logger)<EOL>

Registers more action_logger function callback for post-execution.
This function callback is expected to be called with keyword args.
For more about the arguments that is being passed to the callback,
refer to airflow.utils.cli.action_logging()
:param action_logger: An action logger function
:return: None


logging.debug("<STR_LIT>", __pre_exec_callbacks)<EOL>for cb in __pre_exec_callbacks:<EOL><INDENT>try:<EOL><INDENT>cb(**kwargs)<EOL><DEDENT>except Exception:<EOL><INDENT>logging.exception('<STR_LIT>', cb)<EOL><DEDENT><DEDENT>

Calls callbacks before execution.
Note that any exception from callback will be logged but won't be propagated.
:param kwargs:
:return: None


logging.debug("<STR_LIT>", __post_exec_callbacks)<EOL>for cb in __post_exec_callbacks:<EOL><INDENT>try:<EOL><INDENT>cb(**kwargs)<EOL><DEDENT>except Exception:<EOL><INDENT>logging.exception('<STR_LIT>', cb)<EOL><DEDENT><DEDENT>

Calls callbacks after execution.
As it's being called after execution, it can capture status of execution,
duration, etc. Note that any exception from callback will be logged but
won't be propagated.
:param kwargs:
:return: None


with create_session() as session:<EOL><INDENT>session.add(log)<EOL><DEDENT>

A default action logger callback that behave same as www.utils.action_logging
which uses global session and pushes log ORM object.
:param log: An log ORM instance
:param **_: other keyword arguments that is not being used by this function
:return: None


return [<EOL>cls.SUCCESS,<EOL>cls.FAILED,<EOL>cls.SKIPPED,<EOL>]<EOL>

A list of states indicating that a task started and completed a
run attempt. Note that the attempt could have resulted in failure or
have been interrupted; in any case, it is no longer running.


return [<EOL>cls.NONE,<EOL>cls.SCHEDULED,<EOL>cls.QUEUED,<EOL>cls.RUNNING,<EOL>cls.SHUTDOWN,<EOL>cls.UP_FOR_RETRY,<EOL>cls.UP_FOR_RESCHEDULE<EOL>]<EOL>

A list of states indicating that a task either has not completed
a run or has not even started.


def uncompress_file(input_file_name, file_extension, dest_dir):

if file_extension.lower() not in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>raise NotImplementedError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>.format(file_extension))<EOL><DEDENT>if file_extension.lower() == '<STR_LIT>':<EOL><INDENT>fmodule = gzip.GzipFile<EOL><DEDENT>elif file_extension.lower() == '<STR_LIT>':<EOL><INDENT>fmodule = bz2.BZ2File<EOL><DEDENT>with fmodule(input_file_name, mode='<STR_LIT:rb>') as f_compressed,NamedTemporaryFile(dir=dest_dir,<EOL>mode='<STR_LIT:wb>',<EOL>delete=False) as f_uncompressed:<EOL><INDENT>shutil.copyfileobj(f_compressed, f_uncompressed)<EOL><DEDENT>return f_uncompressed.name<EOL>


<EOL>sig_cache = signature(func)<EOL>non_optional_args = {<EOL>name for (name, param) in sig_cache.parameters.items()<EOL>if param.default == param.empty and<EOL>param.name != '<STR_LIT>' and<EOL>param.kind not in (param.VAR_POSITIONAL, param.VAR_KEYWORD)}<EOL>@wraps(func)<EOL>def wrapper(*args, **kwargs):<EOL><INDENT>if len(args) > <NUM_LIT:1>:<EOL><INDENT>raise AirflowException(<EOL>"<STR_LIT>")<EOL><DEDENT>dag_args = {}<EOL>dag_params = {}<EOL>dag = kwargs.get('<STR_LIT>', None) or settings.CONTEXT_MANAGER_DAG<EOL>if dag:<EOL><INDENT>dag_args = copy(dag.default_args) or {}<EOL>dag_params = copy(dag.params) or {}<EOL><DEDENT>params = {}<EOL>if '<STR_LIT>' in kwargs:<EOL><INDENT>params = kwargs['<STR_LIT>']<EOL><DEDENT>dag_params.update(params)<EOL>default_args = {}<EOL>if '<STR_LIT>' in kwargs:<EOL><INDENT>default_args = kwargs['<STR_LIT>']<EOL>if '<STR_LIT>' in default_args:<EOL><INDENT>dag_params.update(default_args['<STR_LIT>'])<EOL>del default_args['<STR_LIT>']<EOL><DEDENT><DEDENT>dag_args.update(default_args)<EOL>default_args = dag_args<EOL>for arg in sig_cache.parameters:<EOL><INDENT>if arg not in kwargs and arg in default_args:<EOL><INDENT>kwargs[arg] = default_args[arg]<EOL><DEDENT><DEDENT>missing_args = list(non_optional_args - set(kwargs))<EOL>if missing_args:<EOL><INDENT>msg = "<STR_LIT>".format(missing_args)<EOL>raise AirflowException(msg)<EOL><DEDENT>kwargs['<STR_LIT>'] = dag_params<EOL>result = func(*args, **kwargs)<EOL>return result<EOL><DEDENT>return wrapper<EOL>

Function decorator that Looks for an argument named "default_args", and
fills the unspecified arguments from it.

Since python2.* isn't clear about which arguments are missing when
calling a function, and that this can be quite confusing with multi-level
inheritance and argument defaults, this decorator also alerts with
specific information about the missing arguments.


db = self.get_connection(self.presto_conn_id)<EOL>reqkwargs = None<EOL>if db.password is not None:<EOL><INDENT>reqkwargs = {'<STR_LIT>': HTTPBasicAuth(db.login, db.password)}<EOL><DEDENT>return presto.connect(<EOL>host=db.host,<EOL>port=db.port,<EOL>username=db.login,<EOL>source=db.extra_dejson.get('<STR_LIT:source>', '<STR_LIT>'),<EOL>protocol=db.extra_dejson.get('<STR_LIT>', '<STR_LIT:http>'),<EOL>catalog=db.extra_dejson.get('<STR_LIT>', '<STR_LIT>'),<EOL>requests_kwargs=reqkwargs,<EOL>schema=db.schema)<EOL>


if (hasattr(e, '<STR_LIT:message>') and<EOL>'<STR_LIT>' in e.message and<EOL>'<STR_LIT:message>' in e.message):<EOL><INDENT>return ('<STR_LIT>'.format(<EOL>name=e.message['<STR_LIT>'],<EOL>message=e.message['<STR_LIT:message>']))<EOL><DEDENT>else:<EOL><INDENT>return str(e)<EOL><DEDENT>

Parses some DatabaseError to provide a better error message


def get_records(self, hql, parameters=None):

try:<EOL><INDENT>return super().get_records(<EOL>self._strip_sql(hql), parameters)<EOL><DEDENT>except DatabaseError as e:<EOL><INDENT>raise PrestoException(self._get_pretty_exception_message(e))<EOL><DEDENT>


def get_first(self, hql, parameters=None):

try:<EOL><INDENT>return super().get_first(<EOL>self._strip_sql(hql), parameters)<EOL><DEDENT>except DatabaseError as e:<EOL><INDENT>raise PrestoException(self._get_pretty_exception_message(e))<EOL><DEDENT>

Returns only the first row, regardless of how many rows the query
returns.


def get_pandas_df(self, hql, parameters=None):

import pandas<EOL>cursor = self.get_cursor()<EOL>try:<EOL><INDENT>cursor.execute(self._strip_sql(hql), parameters)<EOL>data = cursor.fetchall()<EOL><DEDENT>except DatabaseError as e:<EOL><INDENT>raise PrestoException(self._get_pretty_exception_message(e))<EOL><DEDENT>column_descriptions = cursor.description<EOL>if data:<EOL><INDENT>df = pandas.DataFrame(data)<EOL>df.columns = [c[<NUM_LIT:0>] for c in column_descriptions]<EOL><DEDENT>else:<EOL><INDENT>df = pandas.DataFrame()<EOL><DEDENT>return df<EOL>

Get a pandas dataframe from a sql query.


return super().run(self._strip_sql(hql), parameters)<EOL>

Execute the statement against Presto. Can be used to create views.


def insert_rows(self, table, rows, target_fields=None):

super().insert_rows(table, rows, target_fields, <NUM_LIT:0>)<EOL>

A generic way to insert a set of tuples into a table.

:param table: Name of the target table
:type table: str
:param rows: The rows to insert into the table
:type rows: iterable of tuples
:param target_fields: The names of the columns to fill in the table
:type target_fields: iterable of strings


db = self.get_connection(getattr(self, self.conn_name_attr))<EOL>return self.connector.connect(<EOL>host=db.host,<EOL>port=db.port,<EOL>username=db.login,<EOL>schema=db.schema)<EOL>


def get_pandas_df(self, sql, parameters=None):

import pandas.io.sql as psql<EOL>with closing(self.get_conn()) as conn:<EOL><INDENT>return psql.read_sql(sql, con=conn, params=parameters)<EOL><DEDENT>

Executes the sql and returns a pandas dataframe

:param sql: the sql statement to be executed (str) or a list of
    sql statements to execute
:type sql: str or list
:param parameters: The parameters to render the SQL query with.
:type parameters: mapping or iterable


def get_records(self, sql, parameters=None):

with closing(self.get_conn()) as conn:<EOL><INDENT>with closing(conn.cursor()) as cur:<EOL><INDENT>if parameters is not None:<EOL><INDENT>cur.execute(sql, parameters)<EOL><DEDENT>else:<EOL><INDENT>cur.execute(sql)<EOL><DEDENT>return cur.fetchall()<EOL><DEDENT><DEDENT>

Executes the sql and returns a set of records.

:param sql: the sql statement to be executed (str) or a list of
    sql statements to execute
:type sql: str or list
:param parameters: The parameters to render the SQL query with.
:type parameters: mapping or iterable


def get_first(self, sql, parameters=None):

with closing(self.get_conn()) as conn:<EOL><INDENT>with closing(conn.cursor()) as cur:<EOL><INDENT>if parameters is not None:<EOL><INDENT>cur.execute(sql, parameters)<EOL><DEDENT>else:<EOL><INDENT>cur.execute(sql)<EOL><DEDENT>return cur.fetchone()<EOL><DEDENT><DEDENT>

Executes the sql and returns the first resulting row.

:param sql: the sql statement to be executed (str) or a list of
    sql statements to execute
:type sql: str or list
:param parameters: The parameters to render the SQL query with.
:type parameters: mapping or iterable


def run(self, sql, autocommit=False, parameters=None):

if isinstance(sql, basestring):<EOL><INDENT>sql = [sql]<EOL><DEDENT>with closing(self.get_conn()) as conn:<EOL><INDENT>if self.supports_autocommit:<EOL><INDENT>self.set_autocommit(conn, autocommit)<EOL><DEDENT>with closing(conn.cursor()) as cur:<EOL><INDENT>for s in sql:<EOL><INDENT>if parameters is not None:<EOL><INDENT>self.log.info("<STR_LIT>".format(s, parameters))<EOL>cur.execute(s, parameters)<EOL><DEDENT>else:<EOL><INDENT>self.log.info(s)<EOL>cur.execute(s)<EOL><DEDENT><DEDENT><DEDENT>if not self.get_autocommit(conn):<EOL><INDENT>conn.commit()<EOL><DEDENT><DEDENT>

Runs a command or a list of commands. Pass a list of sql
statements to the sql parameter to get them to execute
sequentially

:param sql: the sql statement to be executed (str) or a list of
    sql statements to execute
:type sql: str or list
:param autocommit: What to set the connection's autocommit setting to
    before executing the query.
:type autocommit: bool
:param parameters: The parameters to render the SQL query with.
:type parameters: mapping or iterable


def set_autocommit(self, conn, autocommit):

if not self.supports_autocommit and autocommit:<EOL><INDENT>self.log.warn(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>"),<EOL>getattr(self, self.conn_name_attr))<EOL><DEDENT>conn.autocommit = autocommit<EOL>

Sets the autocommit flag on the connection


return getattr(conn, '<STR_LIT>', False) and self.supports_autocommit<EOL>

Get autocommit setting for the provided connection.
Return True if conn.autocommit is set to True.
Return False if conn.autocommit is not set or set to False or conn
does not support autocommit.

:param conn: Connection to get autocommit setting from.
:type conn: connection object.
:return: connection autocommit setting.
:rtype: bool


return self.get_conn().cursor()<EOL>


def insert_rows(self, table, rows, target_fields=None, commit_every=<NUM_LIT:1000>,<EOL>replace=False):

if target_fields:<EOL><INDENT>target_fields = "<STR_LIT:U+002CU+0020>".join(target_fields)<EOL>target_fields = "<STR_LIT>".format(target_fields)<EOL><DEDENT>else:<EOL><INDENT>target_fields = '<STR_LIT>'<EOL><DEDENT>i = <NUM_LIT:0><EOL>with closing(self.get_conn()) as conn:<EOL><INDENT>if self.supports_autocommit:<EOL><INDENT>self.set_autocommit(conn, False)<EOL><DEDENT>conn.commit()<EOL>with closing(conn.cursor()) as cur:<EOL><INDENT>for i, row in enumerate(rows, <NUM_LIT:1>):<EOL><INDENT>lst = []<EOL>for cell in row:<EOL><INDENT>lst.append(self._serialize_cell(cell, conn))<EOL><DEDENT>values = tuple(lst)<EOL>placeholders = ["<STR_LIT:%s>", ] * len(values)<EOL>if not replace:<EOL><INDENT>sql = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>sql = "<STR_LIT>"<EOL><DEDENT>sql += "<STR_LIT>".format(<EOL>table,<EOL>target_fields,<EOL>"<STR_LIT:U+002C>".join(placeholders))<EOL>cur.execute(sql, values)<EOL>if commit_every and i % commit_every == <NUM_LIT:0>:<EOL><INDENT>conn.commit()<EOL>self.log.info(<EOL>"<STR_LIT>", i, table<EOL>)<EOL><DEDENT><DEDENT><DEDENT>conn.commit()<EOL><DEDENT>self.log.info("<STR_LIT>", i)<EOL>

A generic way to insert a set of tuples into a table,
a new transaction is created every commit_every rows

:param table: Name of the target table
:type table: str
:param rows: The rows to insert into the table
:type rows: iterable of tuples
:param target_fields: The names of the columns to fill in the table
:type target_fields: iterable of strings
:param commit_every: The maximum number of rows to insert in one
    transaction. Set to 0 to insert all rows in one transaction.
:type commit_every: int
:param replace: Whether to replace instead of insert
:type replace: bool


if cell is None:<EOL><INDENT>return None<EOL><DEDENT>if isinstance(cell, datetime):<EOL><INDENT>return cell.isoformat()<EOL><DEDENT>return str(cell)<EOL>

Returns the SQL literal of the cell as a string.

:param cell: The cell to insert into the table
:type cell: object
:param conn: The database connection
:type conn: connection object
:return: The serialized cell
:rtype: str


raise NotImplementedError()<EOL>

Dumps a database table into a tab-delimited file

:param table: The name of the source table
:type table: str
:param tmp_file: The path of the target file
:type tmp_file: str


raise NotImplementedError()<EOL>

Loads a tab-delimited file into a database table

:param table: The name of the target table
:type table: str
:param tmp_file: The path of the file to load into the table
:type tmp_file: str


session = requests.Session()<EOL>if self.http_conn_id:<EOL><INDENT>conn = self.get_connection(self.http_conn_id)<EOL>if "<STR_LIT>" in conn.host:<EOL><INDENT>self.base_url = conn.host<EOL><DEDENT>else:<EOL><INDENT>schema = conn.schema if conn.schema else "<STR_LIT:http>"<EOL>self.base_url = schema + "<STR_LIT>" + conn.host<EOL><DEDENT>if conn.port:<EOL><INDENT>self.base_url = self.base_url + "<STR_LIT::>" + str(conn.port)<EOL><DEDENT>if conn.login:<EOL><INDENT>session.auth = (conn.login, conn.password)<EOL><DEDENT>if conn.extra:<EOL><INDENT>try:<EOL><INDENT>session.headers.update(conn.extra_dejson)<EOL><DEDENT>except TypeError:<EOL><INDENT>self.log.warn('<STR_LIT>', conn.host)<EOL><DEDENT><DEDENT><DEDENT>if headers:<EOL><INDENT>session.headers.update(headers)<EOL><DEDENT>return session<EOL>

Returns http session for use with requests

:param headers: additional headers to be passed through as a dictionary
:type headers: dict


def run(self, endpoint, data=None, headers=None, extra_options=None):

extra_options = extra_options or {}<EOL>session = self.get_conn(headers)<EOL>if self.base_url and not self.base_url.endswith('<STR_LIT:/>') andendpoint and not endpoint.startswith('<STR_LIT:/>'):<EOL><INDENT>url = self.base_url + '<STR_LIT:/>' + endpoint<EOL><DEDENT>else:<EOL><INDENT>url = (self.base_url or '<STR_LIT>') + (endpoint or '<STR_LIT>')<EOL><DEDENT>req = None<EOL>if self.method == '<STR_LIT:GET>':<EOL><INDENT>req = requests.Request(self.method,<EOL>url,<EOL>params=data,<EOL>headers=headers)<EOL><DEDENT>elif self.method == '<STR_LIT>':<EOL><INDENT>req = requests.Request(self.method,<EOL>url,<EOL>headers=headers)<EOL><DEDENT>else:<EOL><INDENT>req = requests.Request(self.method,<EOL>url,<EOL>data=data,<EOL>headers=headers)<EOL><DEDENT>prepped_request = session.prepare_request(req)<EOL>self.log.info("<STR_LIT>", self.method, url)<EOL>return self.run_and_check(session, prepped_request, extra_options)<EOL>

Performs the request

:param endpoint: the endpoint to be called i.e. resource/v1/query?
:type endpoint: str
:param data: payload to be uploaded or request parameters
:type data: dict
:param headers: additional headers to be passed through as a dictionary
:type headers: dict
:param extra_options: additional options to be used when executing the request
    i.e. {'check_response': False} to avoid checking raising exceptions on non
    2XX or 3XX status codes
:type extra_options: dict


try:<EOL><INDENT>response.raise_for_status()<EOL><DEDENT>except requests.exceptions.HTTPError:<EOL><INDENT>self.log.error("<STR_LIT>", response.reason)<EOL>if self.method not in ['<STR_LIT:GET>', '<STR_LIT>']:<EOL><INDENT>self.log.error(response.text)<EOL><DEDENT>raise AirflowException(str(response.status_code) + "<STR_LIT::>" + response.reason)<EOL><DEDENT>

Checks the status code and raise an AirflowException exception on non 2XX or 3XX
status codes

:param response: A requests response object
:type response: requests.response


def run_and_check(self, session, prepped_request, extra_options):

extra_options = extra_options or {}<EOL>try:<EOL><INDENT>response = session.send(<EOL>prepped_request,<EOL>stream=extra_options.get("<STR_LIT>", False),<EOL>verify=extra_options.get("<STR_LIT>", True),<EOL>proxies=extra_options.get("<STR_LIT>", {}),<EOL>cert=extra_options.get("<STR_LIT>"),<EOL>timeout=extra_options.get("<STR_LIT>"),<EOL>allow_redirects=extra_options.get("<STR_LIT>", True))<EOL>if extra_options.get('<STR_LIT>', True):<EOL><INDENT>self.check_response(response)<EOL><DEDENT>return response<EOL><DEDENT>except requests.exceptions.ConnectionError as ex:<EOL><INDENT>self.log.warn(str(ex) + '<STR_LIT>')<EOL>raise ex<EOL><DEDENT>

Grabs extra options like timeout and actually runs the request,
checking for the result

:param session: the session to be used to execute the request
:type session: requests.Session
:param prepped_request: the prepared request generated in run()
:type prepped_request: session.prepare_request
:param extra_options: additional options to be used when executing the request
    i.e. {'check_response': False} to avoid checking raising exceptions on non 2XX
    or 3XX status codes
:type extra_options: dict


def run_with_advanced_retry(self, _retry_args, *args, **kwargs):

self._retry_obj = tenacity.Retrying(<EOL>**_retry_args<EOL>)<EOL>self._retry_obj(self.run, *args, **kwargs)<EOL>

Runs Hook.run() with a Tenacity decorator attached to it. This is useful for
connectors which might be disturbed by intermittent issues and should not
instantly fail.

:param _retry_args: Arguments which define the retry behaviour.
    See Tenacity documentation at https://github.com/jd/tenacity
:type _retry_args: dict

:Example::

    hook = HttpHook(http_conn_id='my_conn',method='GET')
    retry_args = dict(
         wait=tenacity.wait_exponential(),
         stop=tenacity.stop_after_attempt(10),
         retry=requests.exceptions.ConnectionError
     )
     hook.run_with_advanced_retry(
             endpoint='v1/test',
             _retry_args=retry_args
         )


conn = self.get_connection(self.oracle_conn_id)<EOL>conn_config = {<EOL>'<STR_LIT:user>': conn.login,<EOL>'<STR_LIT:password>': conn.password<EOL>}<EOL>dsn = conn.extra_dejson.get('<STR_LIT>', None)<EOL>sid = conn.extra_dejson.get('<STR_LIT>', None)<EOL>mod = conn.extra_dejson.get('<STR_LIT>', None)<EOL>service_name = conn.extra_dejson.get('<STR_LIT>', None)<EOL>port = conn.port if conn.port else <NUM_LIT><EOL>if dsn and sid and not service_name:<EOL><INDENT>conn_config['<STR_LIT>'] = cx_Oracle.makedsn(dsn, port, sid)<EOL><DEDENT>elif dsn and service_name and not sid:<EOL><INDENT>conn_config['<STR_LIT>'] = cx_Oracle.makedsn(dsn, port, service_name=service_name)<EOL><DEDENT>else:<EOL><INDENT>conn_config['<STR_LIT>'] = conn.host<EOL><DEDENT>if '<STR_LIT>' in conn.extra_dejson:<EOL><INDENT>conn_config['<STR_LIT>'] = conn.extra_dejson.get('<STR_LIT>')<EOL>if '<STR_LIT>' not in conn.extra_dejson:<EOL><INDENT>conn_config['<STR_LIT>'] = conn.extra_dejson.get('<STR_LIT>')<EOL><DEDENT><DEDENT>if '<STR_LIT>' in conn.extra_dejson:<EOL><INDENT>conn_config['<STR_LIT>'] = conn.extra_dejson.get('<STR_LIT>')<EOL><DEDENT>if '<STR_LIT>' in conn.extra_dejson:<EOL><INDENT>conn_config['<STR_LIT>'] = conn.extra_dejson.get('<STR_LIT>')<EOL><DEDENT>if '<STR_LIT>' in conn.extra_dejson:<EOL><INDENT>conn_config['<STR_LIT>'] = conn.extra_dejson.get('<STR_LIT>')<EOL><DEDENT>mode = conn.extra_dejson.get('<STR_LIT>', '<STR_LIT>').lower()<EOL>if mode == '<STR_LIT>':<EOL><INDENT>conn_config['<STR_LIT>'] = cx_Oracle.SYSDBA<EOL><DEDENT>elif mode == '<STR_LIT>':<EOL><INDENT>conn_config['<STR_LIT>'] = cx_Oracle.SYSASM<EOL><DEDENT>elif mode == '<STR_LIT>':<EOL><INDENT>conn_config['<STR_LIT>'] = cx_Oracle.SYSOPER<EOL><DEDENT>elif mode == '<STR_LIT>':<EOL><INDENT>conn_config['<STR_LIT>'] = cx_Oracle.SYSBKP<EOL><DEDENT>elif mode == '<STR_LIT>':<EOL><INDENT>conn_config['<STR_LIT>'] = cx_Oracle.SYSDGD<EOL><DEDENT>elif mode == '<STR_LIT>':<EOL><INDENT>conn_config['<STR_LIT>'] = cx_Oracle.SYSKMT<EOL><DEDENT>elif mode == '<STR_LIT>':<EOL><INDENT>conn_config['<STR_LIT>'] = cx_Oracle.SYSRAC<EOL><DEDENT>purity = conn.extra_dejson.get('<STR_LIT>', '<STR_LIT>').lower()<EOL>if purity == '<STR_LIT>':<EOL><INDENT>conn_config['<STR_LIT>'] = cx_Oracle.ATTR_PURITY_NEW<EOL><DEDENT>elif purity == '<STR_LIT>':<EOL><INDENT>conn_config['<STR_LIT>'] = cx_Oracle.ATTR_PURITY_SELF<EOL><DEDENT>elif purity == '<STR_LIT:default>':<EOL><INDENT>conn_config['<STR_LIT>'] = cx_Oracle.ATTR_PURITY_DEFAULT<EOL><DEDENT>conn = cx_Oracle.connect(**conn_config)<EOL>if mod is not None:<EOL><INDENT>conn.module = mod<EOL><DEDENT>return conn<EOL>

Returns a oracle connection object
Optional parameters for using a custom DSN connection
(instead of using a server alias from tnsnames.ora)
The dsn (data source name) is the TNS entry
(from the Oracle names server or tnsnames.ora file)
or is a string like the one returned from makedsn().

:param dsn: the host address for the Oracle server
:param service_name: the db_unique_name of the database
      that you are connecting to (CONNECT_DATA part of TNS)

You can set these parameters in the extra fields of your connection
as in ``{ "dsn":"some.host.address" , "service_name":"some.service.name" }``
see more param detail in
`cx_Oracle.connect <https://cx-oracle.readthedocs.io/en/latest/module.html#cx_Oracle.connect>`_


def insert_rows(self, table, rows, target_fields=None, commit_every=<NUM_LIT:1000>):

if target_fields:<EOL><INDENT>target_fields = '<STR_LIT:U+002CU+0020>'.join(target_fields)<EOL>target_fields = '<STR_LIT>'.format(target_fields)<EOL><DEDENT>else:<EOL><INDENT>target_fields = '<STR_LIT>'<EOL><DEDENT>conn = self.get_conn()<EOL>cur = conn.cursor()<EOL>if self.supports_autocommit:<EOL><INDENT>cur.execute('<STR_LIT>')<EOL><DEDENT>conn.commit()<EOL>i = <NUM_LIT:0><EOL>for row in rows:<EOL><INDENT>i += <NUM_LIT:1><EOL>lst = []<EOL>for cell in row:<EOL><INDENT>if isinstance(cell, basestring):<EOL><INDENT>lst.append("<STR_LIT:'>" + str(cell).replace("<STR_LIT:'>", "<STR_LIT>") + "<STR_LIT:'>")<EOL><DEDENT>elif cell is None:<EOL><INDENT>lst.append('<STR_LIT>')<EOL><DEDENT>elif type(cell) == float andnumpy.isnan(cell):  <EOL><INDENT>lst.append('<STR_LIT>')<EOL><DEDENT>elif isinstance(cell, numpy.datetime64):<EOL><INDENT>lst.append("<STR_LIT:'>" + str(cell) + "<STR_LIT:'>")<EOL><DEDENT>elif isinstance(cell, datetime):<EOL><INDENT>lst.append("<STR_LIT>" +<EOL>cell.strftime('<STR_LIT>') +<EOL>"<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>lst.append(str(cell))<EOL><DEDENT><DEDENT>values = tuple(lst)<EOL>sql = '<STR_LIT>''<STR_LIT>'.format(table,<EOL>target_fields,<EOL>'<STR_LIT:U+002C>'.join(values))<EOL>cur.execute(sql)<EOL>if i % commit_every == <NUM_LIT:0>:<EOL><INDENT>conn.commit()<EOL>self.log.info('<STR_LIT>', i, table)<EOL><DEDENT><DEDENT>conn.commit()<EOL>cur.close()<EOL>conn.close()<EOL>self.log.info('<STR_LIT>', i)<EOL>

A generic way to insert a set of tuples into a table,
the whole set of inserts is treated as one transaction
Changes from standard DbApiHook implementation:

- Oracle SQL queries in cx_Oracle can not be terminated with a semicolon (`;`)
- Replace NaN values with NULL using `numpy.nan_to_num` (not using
  `is_nan()` because of input types error for strings)
- Coerce datetime cells to Oracle DATETIME format during insert

:param table: target Oracle table, use dot notation to target a
    specific database
:type table: str
:param rows: the rows to insert into the table
:type rows: iterable of tuples
:param target_fields: the names of the columns to fill in the table
:type target_fields: iterable of str
:param commit_every: the maximum number of rows to insert in one transaction
    Default 1000, Set greater than 0.
    Set 1 to insert each row in each single transaction
:type commit_every: int


def bulk_insert_rows(self, table, rows, target_fields=None, commit_every=<NUM_LIT>):

if not rows:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>conn = self.get_conn()<EOL>cursor = conn.cursor()<EOL>values_base = target_fields if target_fields else rows[<NUM_LIT:0>]<EOL>prepared_stm = '<STR_LIT>'.format(<EOL>tablename=table,<EOL>columns='<STR_LIT>'.format('<STR_LIT:U+002CU+0020>'.join(target_fields)) if target_fields else '<STR_LIT>',<EOL>values='<STR_LIT:U+002CU+0020>'.join('<STR_LIT>' % i for i in range(<NUM_LIT:1>, len(values_base) + <NUM_LIT:1>)),<EOL>)<EOL>row_count = <NUM_LIT:0><EOL>row_chunk = []<EOL>for row in rows:<EOL><INDENT>row_chunk.append(row)<EOL>row_count += <NUM_LIT:1><EOL>if row_count % commit_every == <NUM_LIT:0>:<EOL><INDENT>cursor.prepare(prepared_stm)<EOL>cursor.executemany(None, row_chunk)<EOL>conn.commit()<EOL>self.log.info('<STR_LIT>', table, row_count)<EOL>row_chunk = []<EOL><DEDENT><DEDENT>cursor.prepare(prepared_stm)<EOL>cursor.executemany(None, row_chunk)<EOL>conn.commit()<EOL>self.log.info('<STR_LIT>', table, row_count)<EOL>cursor.close()<EOL>conn.close()<EOL>

A performant bulk insert for cx_Oracle
that uses prepared statements via `executemany()`.
For best performance, pass in `rows` as an iterator.

:param table: target Oracle table, use dot notation to target a
    specific database
:type table: str
:param rows: the rows to insert into the table
:type rows: iterable of tuples
:param target_fields: the names of the columns to fill in the table, default None.
    If None, each rows should have some order as table columns name
:type target_fields: iterable of str Or None
:param commit_every: the maximum number of rows to insert in one transaction
    Default 5000. Set greater than 0. Set 1 to insert each row in each transaction
:type commit_every: int


try:<EOL><INDENT>self.get_conn().head_bucket(Bucket=bucket_name)<EOL>return True<EOL><DEDENT>except ClientError as e:<EOL><INDENT>self.log.info(e.response["<STR_LIT>"]["<STR_LIT>"])<EOL>return False<EOL><DEDENT>

Check if bucket_name exists.

:param bucket_name: the name of the bucket
:type bucket_name: str


s3 = self.get_resource_type('<STR_LIT>')<EOL>return s3.Bucket(bucket_name)<EOL>

Returns a boto3.S3.Bucket object

:param bucket_name: the name of the bucket
:type bucket_name: str


def create_bucket(self, bucket_name, region_name=None):

s3_conn = self.get_conn()<EOL>if not region_name:<EOL><INDENT>region_name = s3_conn.meta.region_name<EOL><DEDENT>if region_name == '<STR_LIT>':<EOL><INDENT>self.get_conn().create_bucket(Bucket=bucket_name)<EOL><DEDENT>else:<EOL><INDENT>self.get_conn().create_bucket(Bucket=bucket_name,<EOL>CreateBucketConfiguration={<EOL>'<STR_LIT>': region_name<EOL>})<EOL><DEDENT>

Creates an Amazon S3 bucket.

:param bucket_name: The name of the bucket
:type bucket_name: str
:param region_name: The name of the aws region in which to create the bucket.
:type region_name: str


def check_for_prefix(self, bucket_name, prefix, delimiter):

prefix = prefix + delimiter if prefix[-<NUM_LIT:1>] != delimiter else prefix<EOL>prefix_split = re.split(r'<STR_LIT>'.format(d=delimiter), prefix, <NUM_LIT:1>)<EOL>previous_level = prefix_split[<NUM_LIT:0>]<EOL>plist = self.list_prefixes(bucket_name, previous_level, delimiter)<EOL>return False if plist is None else prefix in plist<EOL>

Checks that a prefix exists in a bucket

:param bucket_name: the name of the bucket
:type bucket_name: str
:param prefix: a key prefix
:type prefix: str
:param delimiter: the delimiter marks key hierarchy.
:type delimiter: str


def list_prefixes(self, bucket_name, prefix='<STR_LIT>', delimiter='<STR_LIT>',<EOL>page_size=None, max_items=None):

config = {<EOL>'<STR_LIT>': page_size,<EOL>'<STR_LIT>': max_items,<EOL>}<EOL>paginator = self.get_conn().get_paginator('<STR_LIT>')<EOL>response = paginator.paginate(Bucket=bucket_name,<EOL>Prefix=prefix,<EOL>Delimiter=delimiter,<EOL>PaginationConfig=config)<EOL>has_results = False<EOL>prefixes = []<EOL>for page in response:<EOL><INDENT>if '<STR_LIT>' in page:<EOL><INDENT>has_results = True<EOL>for p in page['<STR_LIT>']:<EOL><INDENT>prefixes.append(p['<STR_LIT>'])<EOL><DEDENT><DEDENT><DEDENT>if has_results:<EOL><INDENT>return prefixes<EOL><DEDENT>

Lists prefixes in a bucket under prefix

:param bucket_name: the name of the bucket
:type bucket_name: str
:param prefix: a key prefix
:type prefix: str
:param delimiter: the delimiter marks key hierarchy.
:type delimiter: str
:param page_size: pagination size
:type page_size: int
:param max_items: maximum items to return
:type max_items: int


def list_keys(self, bucket_name, prefix='<STR_LIT>', delimiter='<STR_LIT>',<EOL>page_size=None, max_items=None):

config = {<EOL>'<STR_LIT>': page_size,<EOL>'<STR_LIT>': max_items,<EOL>}<EOL>paginator = self.get_conn().get_paginator('<STR_LIT>')<EOL>response = paginator.paginate(Bucket=bucket_name,<EOL>Prefix=prefix,<EOL>Delimiter=delimiter,<EOL>PaginationConfig=config)<EOL>has_results = False<EOL>keys = []<EOL>for page in response:<EOL><INDENT>if '<STR_LIT>' in page:<EOL><INDENT>has_results = True<EOL>for k in page['<STR_LIT>']:<EOL><INDENT>keys.append(k['<STR_LIT>'])<EOL><DEDENT><DEDENT><DEDENT>if has_results:<EOL><INDENT>return keys<EOL><DEDENT>

Lists keys in a bucket under prefix and not containing delimiter

:param bucket_name: the name of the bucket
:type bucket_name: str
:param prefix: a key prefix
:type prefix: str
:param delimiter: the delimiter marks key hierarchy.
:type delimiter: str
:param page_size: pagination size
:type page_size: int
:param max_items: maximum items to return
:type max_items: int


def check_for_key(self, key, bucket_name=None):

if not bucket_name:<EOL><INDENT>(bucket_name, key) = self.parse_s3_url(key)<EOL><DEDENT>try:<EOL><INDENT>self.get_conn().head_object(Bucket=bucket_name, Key=key)<EOL>return True<EOL><DEDENT>except ClientError as e:<EOL><INDENT>self.log.info(e.response["<STR_LIT>"]["<STR_LIT>"])<EOL>return False<EOL><DEDENT>

Checks if a key exists in a bucket

:param key: S3 key that will point to the file
:type key: str
:param bucket_name: Name of the bucket in which the file is stored
:type bucket_name: str


def get_key(self, key, bucket_name=None):

if not bucket_name:<EOL><INDENT>(bucket_name, key) = self.parse_s3_url(key)<EOL><DEDENT>obj = self.get_resource_type('<STR_LIT>').Object(bucket_name, key)<EOL>obj.load()<EOL>return obj<EOL>

Returns a boto3.s3.Object

:param key: the path to the key
:type key: str
:param bucket_name: the name of the bucket
:type bucket_name: str


def read_key(self, key, bucket_name=None):

obj = self.get_key(key, bucket_name)<EOL>return obj.get()['<STR_LIT>'].read().decode('<STR_LIT:utf-8>')<EOL>

Reads a key from S3

:param key: S3 key that will point to the file
:type key: str
:param bucket_name: Name of the bucket in which the file is stored
:type bucket_name: str


if input_serialization is None:<EOL><INDENT>input_serialization = {'<STR_LIT>': {}}<EOL><DEDENT>if output_serialization is None:<EOL><INDENT>output_serialization = {'<STR_LIT>': {}}<EOL><DEDENT>if not bucket_name:<EOL><INDENT>(bucket_name, key) = self.parse_s3_url(key)<EOL><DEDENT>response = self.get_conn().select_object_content(<EOL>Bucket=bucket_name,<EOL>Key=key,<EOL>Expression=expression,<EOL>ExpressionType=expression_type,<EOL>InputSerialization=input_serialization,<EOL>OutputSerialization=output_serialization)<EOL>return '<STR_LIT>'.join(event['<STR_LIT>']['<STR_LIT>'].decode('<STR_LIT:utf-8>')<EOL>for event in response['<STR_LIT>']<EOL>if '<STR_LIT>' in event)<EOL>

Reads a key with S3 Select.

:param key: S3 key that will point to the file
:type key: str
:param bucket_name: Name of the bucket in which the file is stored
:type bucket_name: str
:param expression: S3 Select expression
:type expression: str
:param expression_type: S3 Select expression type
:type expression_type: str
:param input_serialization: S3 Select input data serialization format
:type input_serialization: dict
:param output_serialization: S3 Select output data serialization format
:type output_serialization: dict
:return: retrieved subset of original data by S3 Select
:rtype: str

.. seealso::
    For more details about S3 Select parameters:
    http://boto3.readthedocs.io/en/latest/reference/services/s3.html#S3.Client.select_object_content


def check_for_wildcard_key(self,<EOL>wildcard_key, bucket_name=None, delimiter='<STR_LIT>'):

return self.get_wildcard_key(wildcard_key=wildcard_key,<EOL>bucket_name=bucket_name,<EOL>delimiter=delimiter) is not None<EOL>

Checks that a key matching a wildcard expression exists in a bucket

:param wildcard_key: the path to the key
:type wildcard_key: str
:param bucket_name: the name of the bucket
:type bucket_name: str
:param delimiter: the delimiter marks key hierarchy
:type delimiter: str


def get_wildcard_key(self, wildcard_key, bucket_name=None, delimiter='<STR_LIT>'):

if not bucket_name:<EOL><INDENT>(bucket_name, wildcard_key) = self.parse_s3_url(wildcard_key)<EOL><DEDENT>prefix = re.split(r'<STR_LIT>', wildcard_key, <NUM_LIT:1>)[<NUM_LIT:0>]<EOL>klist = self.list_keys(bucket_name, prefix=prefix, delimiter=delimiter)<EOL>if klist:<EOL><INDENT>key_matches = [k for k in klist if fnmatch.fnmatch(k, wildcard_key)]<EOL>if key_matches:<EOL><INDENT>return self.get_key(key_matches[<NUM_LIT:0>], bucket_name)<EOL><DEDENT><DEDENT>

Returns a boto3.s3.Object object matching the wildcard expression

:param wildcard_key: the path to the key
:type wildcard_key: str
:param bucket_name: the name of the bucket
:type bucket_name: str
:param delimiter: the delimiter marks key hierarchy
:type delimiter: str


if not bucket_name:<EOL><INDENT>(bucket_name, key) = self.parse_s3_url(key)<EOL><DEDENT>if not replace and self.check_for_key(key, bucket_name):<EOL><INDENT>raise ValueError("<STR_LIT>".format(key=key))<EOL><DEDENT>extra_args = {}<EOL>if encrypt:<EOL><INDENT>extra_args['<STR_LIT>'] = "<STR_LIT>"<EOL><DEDENT>client = self.get_conn()<EOL>client.upload_file(filename, bucket_name, key, ExtraArgs=extra_args)<EOL>

Loads a local file to S3

:param filename: name of the file to load.
:type filename: str
:param key: S3 key that will point to the file
:type key: str
:param bucket_name: Name of the bucket in which to store the file
:type bucket_name: str
:param replace: A flag to decide whether or not to overwrite the key
    if it already exists. If replace is False and the key exists, an
    error will be raised.
:type replace: bool
:param encrypt: If True, the file will be encrypted on the server-side
    by S3 and will be stored in an encrypted form while at rest in S3.
:type encrypt: bool


self.load_bytes(string_data.encode(encoding),<EOL>key=key,<EOL>bucket_name=bucket_name,<EOL>replace=replace,<EOL>encrypt=encrypt)<EOL>

Loads a string to S3

This is provided as a convenience to drop a string in S3. It uses the
boto infrastructure to ship a file to s3.

:param string_data: str to set as content for the key.
:type string_data: str
:param key: S3 key that will point to the file
:type key: str
:param bucket_name: Name of the bucket in which to store the file
:type bucket_name: str
:param replace: A flag to decide whether or not to overwrite the key
    if it already exists
:type replace: bool
:param encrypt: If True, the file will be encrypted on the server-side
    by S3 and will be stored in an encrypted form while at rest in S3.
:type encrypt: bool


if not bucket_name:<EOL><INDENT>(bucket_name, key) = self.parse_s3_url(key)<EOL><DEDENT>if not replace and self.check_for_key(key, bucket_name):<EOL><INDENT>raise ValueError("<STR_LIT>".format(key=key))<EOL><DEDENT>extra_args = {}<EOL>if encrypt:<EOL><INDENT>extra_args['<STR_LIT>'] = "<STR_LIT>"<EOL><DEDENT>filelike_buffer = BytesIO(bytes_data)<EOL>client = self.get_conn()<EOL>client.upload_fileobj(filelike_buffer, bucket_name, key, ExtraArgs=extra_args)<EOL>

Loads bytes to S3

This is provided as a convenience to drop a string in S3. It uses the
boto infrastructure to ship a file to s3.

:param bytes_data: bytes to set as content for the key.
:type bytes_data: bytes
:param key: S3 key that will point to the file
:type key: str
:param bucket_name: Name of the bucket in which to store the file
:type bucket_name: str
:param replace: A flag to decide whether or not to overwrite the key
    if it already exists
:type replace: bool
:param encrypt: If True, the file will be encrypted on the server-side
    by S3 and will be stored in an encrypted form while at rest in S3.
:type encrypt: bool


if not bucket_name:<EOL><INDENT>(bucket_name, key) = self.parse_s3_url(key)<EOL><DEDENT>if not replace and self.check_for_key(key, bucket_name):<EOL><INDENT>raise ValueError("<STR_LIT>".format(key=key))<EOL><DEDENT>extra_args = {}<EOL>if encrypt:<EOL><INDENT>extra_args['<STR_LIT>'] = "<STR_LIT>"<EOL><DEDENT>client = self.get_conn()<EOL>client.upload_fileobj(file_obj, bucket_name, key, ExtraArgs=extra_args)<EOL>

Loads a file object to S3

:param file_obj: The file-like object to set as the content for the S3 key.
:type file_obj: file-like object
:param key: S3 key that will point to the file
:type key: str
:param bucket_name: Name of the bucket in which to store the file
:type bucket_name: str
:param replace: A flag that indicates whether to overwrite the key
    if it already exists.
:type replace: bool
:param encrypt: If True, S3 encrypts the file on the server,
    and the file is stored in encrypted form at rest in S3.
:type encrypt: bool


if dest_bucket_name is None:<EOL><INDENT>dest_bucket_name, dest_bucket_key = self.parse_s3_url(dest_bucket_key)<EOL><DEDENT>else:<EOL><INDENT>parsed_url = urlparse(dest_bucket_key)<EOL>if parsed_url.scheme != '<STR_LIT>' or parsed_url.netloc != '<STR_LIT>':<EOL><INDENT>raise AirflowException('<STR_LIT>' +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>if source_bucket_name is None:<EOL><INDENT>source_bucket_name, source_bucket_key = self.parse_s3_url(source_bucket_key)<EOL><DEDENT>else:<EOL><INDENT>parsed_url = urlparse(source_bucket_key)<EOL>if parsed_url.scheme != '<STR_LIT>' or parsed_url.netloc != '<STR_LIT>':<EOL><INDENT>raise AirflowException('<STR_LIT>' +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>CopySource = {'<STR_LIT>': source_bucket_name,<EOL>'<STR_LIT>': source_bucket_key,<EOL>'<STR_LIT>': source_version_id}<EOL>response = self.get_conn().copy_object(Bucket=dest_bucket_name,<EOL>Key=dest_bucket_key,<EOL>CopySource=CopySource)<EOL>return response<EOL>

Creates a copy of an object that is already stored in S3.

Note: the S3 connection used here needs to have access to both
source and destination bucket/key.

:param source_bucket_key: The key of the source object.

    It can be either full s3:// style url or relative path from root level.

    When it's specified as a full s3:// url, please omit source_bucket_name.
:type source_bucket_key: str
:param dest_bucket_key: The key of the object to copy to.

    The convention to specify `dest_bucket_key` is the same
    as `source_bucket_key`.
:type dest_bucket_key: str
:param source_bucket_name: Name of the S3 bucket where the source object is in.

    It should be omitted when `source_bucket_key` is provided as a full s3:// url.
:type source_bucket_name: str
:param dest_bucket_name: Name of the S3 bucket to where the object is copied.

    It should be omitted when `dest_bucket_key` is provided as a full s3:// url.
:type dest_bucket_name: str
:param source_version_id: Version ID of the source object (OPTIONAL)
:type source_version_id: str


if isinstance(keys, list):<EOL><INDENT>keys = keys<EOL><DEDENT>else:<EOL><INDENT>keys = [keys]<EOL><DEDENT>delete_dict = {"<STR_LIT>": [{"<STR_LIT>": k} for k in keys]}<EOL>response = self.get_conn().delete_objects(Bucket=bucket,<EOL>Delete=delete_dict)<EOL>return response<EOL>

:param bucket: Name of the bucket in which you are going to delete object(s)
:type bucket: str
:param keys: The key(s) to delete from S3 bucket.

    When ``keys`` is a string, it's supposed to be the key name of
    the single object to delete.

    When ``keys`` is a list, it's supposed to be the list of the
    keys to delete.
:type keys: str or list


conn.autocommit(autocommit)<EOL>

MySql connection sets autocommit in a different way.


return conn.get_autocommit()<EOL>

MySql connection gets autocommit in a different way.

:param conn: connection to get autocommit setting from.
:type conn: connection object.
:return: connection autocommit setting
:rtype: bool